2024-11-13 19:23:06,982 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 19:23:06,994 main DEBUG Took 0.009865 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 19:23:06,994 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 19:23:06,994 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 19:23:06,995 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 19:23:06,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,005 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 19:23:07,022 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,024 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,024 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,026 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,026 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,028 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,029 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,031 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,032 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,032 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,034 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,034 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,035 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,035 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,036 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,036 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,037 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,037 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,038 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,039 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 19:23:07,039 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,040 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 19:23:07,042 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 19:23:07,044 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 19:23:07,046 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 19:23:07,047 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 19:23:07,049 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 19:23:07,049 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 19:23:07,059 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 19:23:07,061 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 19:23:07,063 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 19:23:07,063 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 19:23:07,063 main DEBUG createAppenders(={Console}) 2024-11-13 19:23:07,064 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-13 19:23:07,065 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-13 19:23:07,065 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-13 19:23:07,066 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 19:23:07,066 main DEBUG OutputStream closed 2024-11-13 19:23:07,067 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 19:23:07,067 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 19:23:07,067 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-13 19:23:07,154 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 19:23:07,157 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 19:23:07,158 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 19:23:07,160 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 19:23:07,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 19:23:07,161 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 19:23:07,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 19:23:07,162 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 19:23:07,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 19:23:07,163 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 19:23:07,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 19:23:07,164 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 19:23:07,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 19:23:07,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 19:23:07,165 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 19:23:07,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 19:23:07,166 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 19:23:07,168 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 19:23:07,170 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 19:23:07,170 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-13 19:23:07,171 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 19:23:07,172 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-13T19:23:07,426 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e 2024-11-13 19:23:07,429 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 19:23:07,429 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T19:23:07,438 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-13T19:23:07,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T19:23:07,476 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de, deleteOnExit=true 2024-11-13T19:23:07,477 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T19:23:07,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/test.cache.data in system properties and HBase conf 2024-11-13T19:23:07,478 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T19:23:07,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/hadoop.log.dir in system properties and HBase conf 2024-11-13T19:23:07,479 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T19:23:07,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T19:23:07,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T19:23:07,568 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T19:23:07,657 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T19:23:07,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T19:23:07,662 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T19:23:07,663 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T19:23:07,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T19:23:07,664 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T19:23:07,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T19:23:07,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T19:23:07,665 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T19:23:07,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T19:23:07,666 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/nfs.dump.dir in system properties and HBase conf 2024-11-13T19:23:07,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/java.io.tmpdir in system properties and HBase conf 2024-11-13T19:23:07,667 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T19:23:07,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T19:23:07,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T19:23:08,710 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T19:23:08,803 INFO [Time-limited test {}] log.Log(170): Logging initialized @2571ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T19:23:08,895 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:23:08,962 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:23:08,984 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:23:08,984 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:23:08,986 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T19:23:08,997 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:23:09,000 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:23:09,001 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:23:09,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/java.io.tmpdir/jetty-localhost-44201-hadoop-hdfs-3_4_1-tests_jar-_-any-12434058620594695323/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T19:23:09,216 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:44201} 2024-11-13T19:23:09,216 INFO [Time-limited test {}] server.Server(415): Started @2984ms 2024-11-13T19:23:09,690 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T19:23:09,701 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T19:23:09,702 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T19:23:09,702 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T19:23:09,703 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T19:23:09,703 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/hadoop.log.dir/,AVAILABLE} 2024-11-13T19:23:09,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T19:23:09,817 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/java.io.tmpdir/jetty-localhost-36169-hadoop-hdfs-3_4_1-tests_jar-_-any-9641662020332219388/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:23:09,818 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:36169} 2024-11-13T19:23:09,818 INFO [Time-limited test {}] server.Server(415): Started @3586ms 2024-11-13T19:23:09,873 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T19:23:10,638 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/data/data2/current/BP-1151172645-172.17.0.2-1731525788290/current, will proceed with Du for space computation calculation, 2024-11-13T19:23:10,638 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/data/data1/current/BP-1151172645-172.17.0.2-1731525788290/current, will proceed with Du for space computation calculation, 2024-11-13T19:23:10,670 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T19:23:10,710 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76473bd9169e9420 with lease ID 0x75b0232dec91214a: Processing first storage report for DS-af2a52b2-01a0-47e1-95c9-9a760e7434ba from datanode DatanodeRegistration(127.0.0.1:38613, datanodeUuid=c432c574-f214-4d93-8597-9db924fa2591, infoPort=44019, infoSecurePort=0, ipcPort=36889, storageInfo=lv=-57;cid=testClusterID;nsid=664840194;c=1731525788290) 2024-11-13T19:23:10,712 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76473bd9169e9420 with lease ID 0x75b0232dec91214a: from storage DS-af2a52b2-01a0-47e1-95c9-9a760e7434ba node DatanodeRegistration(127.0.0.1:38613, datanodeUuid=c432c574-f214-4d93-8597-9db924fa2591, infoPort=44019, infoSecurePort=0, ipcPort=36889, storageInfo=lv=-57;cid=testClusterID;nsid=664840194;c=1731525788290), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-13T19:23:10,712 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x76473bd9169e9420 with lease ID 0x75b0232dec91214a: Processing first storage report for DS-1b08bce1-ef7d-4cc8-b182-08ba9d68b4bb from datanode DatanodeRegistration(127.0.0.1:38613, datanodeUuid=c432c574-f214-4d93-8597-9db924fa2591, infoPort=44019, infoSecurePort=0, ipcPort=36889, storageInfo=lv=-57;cid=testClusterID;nsid=664840194;c=1731525788290) 2024-11-13T19:23:10,712 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x76473bd9169e9420 with lease ID 0x75b0232dec91214a: from storage DS-1b08bce1-ef7d-4cc8-b182-08ba9d68b4bb node DatanodeRegistration(127.0.0.1:38613, datanodeUuid=c432c574-f214-4d93-8597-9db924fa2591, infoPort=44019, infoSecurePort=0, ipcPort=36889, storageInfo=lv=-57;cid=testClusterID;nsid=664840194;c=1731525788290), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T19:23:10,754 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e 2024-11-13T19:23:10,816 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/zookeeper_0, clientPort=52667, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T19:23:10,824 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52667 2024-11-13T19:23:10,834 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:23:10,837 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:23:11,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741825_1001 (size=7) 2024-11-13T19:23:11,441 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a with version=8 2024-11-13T19:23:11,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/hbase-staging 2024-11-13T19:23:11,520 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T19:23:11,758 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e4b56bc8c4b3:0 server-side Connection retries=45 2024-11-13T19:23:11,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:23:11,767 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:23:11,772 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:23:11,772 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:23:11,772 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:23:11,926 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T19:23:11,988 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T19:23:11,997 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T19:23:12,001 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:23:12,027 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 123678 (auto-detected) 2024-11-13T19:23:12,028 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T19:23:12,049 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44929 2024-11-13T19:23:12,074 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44929 connecting to ZooKeeper ensemble=127.0.0.1:52667 2024-11-13T19:23:12,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:449290x0, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:23:12,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44929-0x10136f06ec50000 connected 2024-11-13T19:23:12,282 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:23:12,284 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:23:12,293 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:23:12,296 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a, hbase.cluster.distributed=false 2024-11-13T19:23:12,320 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:23:12,324 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-13T19:23:12,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44929 2024-11-13T19:23:12,325 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44929 2024-11-13T19:23:12,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-13T19:23:12,326 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44929 2024-11-13T19:23:12,425 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e4b56bc8c4b3:0 server-side Connection retries=45 2024-11-13T19:23:12,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:23:12,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T19:23:12,427 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T19:23:12,427 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T19:23:12,428 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T19:23:12,431 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T19:23:12,433 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T19:23:12,434 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40371 2024-11-13T19:23:12,436 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40371 connecting to ZooKeeper ensemble=127.0.0.1:52667 2024-11-13T19:23:12,437 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:23:12,441 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:23:12,455 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403710x0, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T19:23:12,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40371-0x10136f06ec50001 connected 2024-11-13T19:23:12,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:23:12,460 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T19:23:12,467 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T19:23:12,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T19:23:12,474 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T19:23:12,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40371 2024-11-13T19:23:12,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40371 2024-11-13T19:23:12,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40371 2024-11-13T19:23:12,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40371 2024-11-13T19:23:12,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40371 2024-11-13T19:23:12,499 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e4b56bc8c4b3:44929 2024-11-13T19:23:12,500 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:12,514 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:23:12,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:23:12,517 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:12,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:12,547 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T19:23:12,547 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:12,548 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T19:23:12,549 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e4b56bc8c4b3,44929,1731525791607 from backup master directory 2024-11-13T19:23:12,555 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:23:12,555 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:12,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T19:23:12,556 WARN [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:23:12,557 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:12,559 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T19:23:12,561 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T19:23:12,621 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/hbase.id] with ID: 4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5 2024-11-13T19:23:12,621 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/.tmp/hbase.id 2024-11-13T19:23:12,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741826_1002 (size=42) 2024-11-13T19:23:13,033 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/.tmp/hbase.id]:[hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/hbase.id] 2024-11-13T19:23:13,080 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T19:23:13,085 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T19:23:13,103 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-13T19:23:13,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:13,139 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:13,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741827_1003 (size=196) 2024-11-13T19:23:13,574 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:23:13,576 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T19:23:13,590 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:13,594 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:23:13,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741828_1004 (size=1189) 2024-11-13T19:23:14,039 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store 2024-11-13T19:23:14,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741829_1005 (size=34) 2024-11-13T19:23:14,468 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T19:23:14,471 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:14,472 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T19:23:14,473 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:23:14,473 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:23:14,474 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T19:23:14,474 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:23:14,475 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:23:14,476 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731525794472Disabling compacts and flushes for region at 1731525794472Disabling writes for close at 1731525794474 (+2 ms)Writing region close event to WAL at 1731525794474Closed at 1731525794474 2024-11-13T19:23:14,478 WARN [master/e4b56bc8c4b3:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/.initializing 2024-11-13T19:23:14,478 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/WALs/e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:14,486 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:23:14,501 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4b56bc8c4b3%2C44929%2C1731525791607, suffix=, logDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/WALs/e4b56bc8c4b3,44929,1731525791607, archiveDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/oldWALs, maxLogs=10 2024-11-13T19:23:14,532 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/WALs/e4b56bc8c4b3,44929,1731525791607/e4b56bc8c4b3%2C44929%2C1731525791607.1731525794506, exclude list is [], retry=0 2024-11-13T19:23:14,550 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38613,DS-af2a52b2-01a0-47e1-95c9-9a760e7434ba,DISK] 2024-11-13T19:23:14,553 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T19:23:14,585 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/WALs/e4b56bc8c4b3,44929,1731525791607/e4b56bc8c4b3%2C44929%2C1731525791607.1731525794506 2024-11-13T19:23:14,586 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44019:44019)] 2024-11-13T19:23:14,587 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:23:14,587 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:14,590 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,591 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T19:23:14,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:14,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:14,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T19:23:14,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:14,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:14,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,664 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T19:23:14,665 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:14,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:14,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T19:23:14,669 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:14,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:14,671 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,675 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,676 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,682 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,683 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,688 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T19:23:14,692 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T19:23:14,696 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:23:14,697 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63221925, jitterRate=-0.057919904589653015}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T19:23:14,704 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731525794602Initializing all the Stores at 1731525794605 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731525794605Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525794606 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525794606Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525794606Cleaning up temporary data from old regions at 1731525794683 (+77 ms)Region opened successfully at 1731525794703 (+20 ms) 2024-11-13T19:23:14,705 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T19:23:14,737 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43709fae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4b56bc8c4b3/172.17.0.2:0 2024-11-13T19:23:14,764 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T19:23:14,775 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T19:23:14,775 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T19:23:14,778 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T19:23:14,779 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T19:23:14,784 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T19:23:14,784 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T19:23:14,808 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T19:23:14,817 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T19:23:14,855 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T19:23:14,859 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T19:23:14,862 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T19:23:14,872 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T19:23:14,874 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T19:23:14,878 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T19:23:14,888 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T19:23:14,890 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T19:23:14,897 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T19:23:14,917 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T19:23:14,921 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T19:23:14,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:23:14,930 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T19:23:14,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:14,930 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:14,933 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e4b56bc8c4b3,44929,1731525791607, sessionid=0x10136f06ec50000, setting cluster-up flag (Was=false) 2024-11-13T19:23:14,955 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:14,955 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:14,980 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T19:23:14,982 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:14,997 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:14,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:15,022 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T19:23:15,025 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:15,035 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T19:23:15,085 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(746): ClusterId : 4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5 2024-11-13T19:23:15,088 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T19:23:15,099 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T19:23:15,099 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T19:23:15,106 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T19:23:15,107 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b68acb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4b56bc8c4b3/172.17.0.2:0 2024-11-13T19:23:15,108 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T19:23:15,117 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T19:23:15,121 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e4b56bc8c4b3:40371 2024-11-13T19:23:15,124 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T19:23:15,124 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T19:23:15,124 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T19:23:15,124 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T19:23:15,127 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(2659): reportForDuty to master=e4b56bc8c4b3,44929,1731525791607 with port=40371, startcode=1731525792389 2024-11-13T19:23:15,131 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e4b56bc8c4b3,44929,1731525791607 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T19:23:15,137 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T19:23:15,140 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e4b56bc8c4b3:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:23:15,140 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e4b56bc8c4b3:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:23:15,141 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e4b56bc8c4b3:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:23:15,141 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e4b56bc8c4b3:0, corePoolSize=5, maxPoolSize=5 2024-11-13T19:23:15,141 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e4b56bc8c4b3:0, corePoolSize=10, maxPoolSize=10 2024-11-13T19:23:15,142 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,142 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e4b56bc8c4b3:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:23:15,142 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,145 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731525825145 2024-11-13T19:23:15,147 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T19:23:15,148 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:23:15,148 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T19:23:15,148 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T19:23:15,151 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T19:23:15,152 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T19:23:15,152 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T19:23:15,152 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T19:23:15,154 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:15,155 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T19:23:15,154 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,159 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T19:23:15,160 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T19:23:15,163 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T19:23:15,167 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T19:23:15,167 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T19:23:15,171 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e4b56bc8c4b3:0:becomeActiveMaster-HFileCleaner.large.0-1731525795168,5,FailOnTimeoutGroup] 2024-11-13T19:23:15,174 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e4b56bc8c4b3:0:becomeActiveMaster-HFileCleaner.small.0-1731525795171,5,FailOnTimeoutGroup] 2024-11-13T19:23:15,175 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,175 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T19:23:15,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741831_1007 (size=1321) 2024-11-13T19:23:15,176 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,177 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57053, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T19:23:15,184 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44929 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:15,187 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44929 {}] master.ServerManager(517): Registering regionserver=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:15,202 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:23:15,202 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45187 2024-11-13T19:23:15,202 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T19:23:15,213 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:23:15,214 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] zookeeper.ZKUtil(111): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:15,214 WARN [RS:0;e4b56bc8c4b3:40371 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T19:23:15,214 INFO [RS:0;e4b56bc8c4b3:40371 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:23:15,215 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/WALs/e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:15,217 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e4b56bc8c4b3,40371,1731525792389] 2024-11-13T19:23:15,243 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T19:23:15,263 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T19:23:15,268 INFO [RS:0;e4b56bc8c4b3:40371 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T19:23:15,268 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,269 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T19:23:15,275 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T19:23:15,276 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,277 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,277 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,277 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,277 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,277 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,278 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e4b56bc8c4b3:0, corePoolSize=2, maxPoolSize=2 2024-11-13T19:23:15,278 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,278 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,278 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,278 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,278 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,278 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e4b56bc8c4b3:0, corePoolSize=1, maxPoolSize=1 2024-11-13T19:23:15,279 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e4b56bc8c4b3:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:23:15,279 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0, corePoolSize=3, maxPoolSize=3 2024-11-13T19:23:15,280 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,280 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,280 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,280 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,280 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,281 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,40371,1731525792389-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:23:15,300 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T19:23:15,302 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,40371,1731525792389-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,302 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,303 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.Replication(171): e4b56bc8c4b3,40371,1731525792389 started 2024-11-13T19:23:15,320 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:15,321 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1482): Serving as e4b56bc8c4b3,40371,1731525792389, RpcServer on e4b56bc8c4b3/172.17.0.2:40371, sessionid=0x10136f06ec50001 2024-11-13T19:23:15,321 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T19:23:15,322 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:15,322 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4b56bc8c4b3,40371,1731525792389' 2024-11-13T19:23:15,322 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T19:23:15,323 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T19:23:15,324 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T19:23:15,324 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T19:23:15,324 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:15,324 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4b56bc8c4b3,40371,1731525792389' 2024-11-13T19:23:15,324 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T19:23:15,325 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T19:23:15,325 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T19:23:15,326 INFO [RS:0;e4b56bc8c4b3:40371 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T19:23:15,326 INFO [RS:0;e4b56bc8c4b3:40371 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T19:23:15,431 INFO [RS:0;e4b56bc8c4b3:40371 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T19:23:15,434 INFO [RS:0;e4b56bc8c4b3:40371 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4b56bc8c4b3%2C40371%2C1731525792389, suffix=, logDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/WALs/e4b56bc8c4b3,40371,1731525792389, archiveDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/oldWALs, maxLogs=32 2024-11-13T19:23:15,450 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/WALs/e4b56bc8c4b3,40371,1731525792389/e4b56bc8c4b3%2C40371%2C1731525792389.1731525795437, exclude list is [], retry=0 2024-11-13T19:23:15,455 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38613,DS-af2a52b2-01a0-47e1-95c9-9a760e7434ba,DISK] 2024-11-13T19:23:15,458 INFO [RS:0;e4b56bc8c4b3:40371 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/WALs/e4b56bc8c4b3,40371,1731525792389/e4b56bc8c4b3%2C40371%2C1731525792389.1731525795437 2024-11-13T19:23:15,459 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44019:44019)] 2024-11-13T19:23:15,575 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T19:23:15,576 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:23:15,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741833_1009 (size=32) 2024-11-13T19:23:15,996 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:16,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T19:23:16,006 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T19:23:16,007 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T19:23:16,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T19:23:16,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,013 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T19:23:16,016 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T19:23:16,017 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,018 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T19:23:16,021 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T19:23:16,021 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,022 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,022 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T19:23:16,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740 2024-11-13T19:23:16,025 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740 2024-11-13T19:23:16,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T19:23:16,028 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T19:23:16,030 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:23:16,033 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T19:23:16,037 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:23:16,038 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61843400, jitterRate=-0.07846152782440186}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:23:16,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731525795997Initializing all the Stores at 1731525796001 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731525796001Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731525796002 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525796002Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731525796002Cleaning up temporary data from old regions at 1731525796028 (+26 ms)Region opened successfully at 1731525796040 (+12 ms) 2024-11-13T19:23:16,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T19:23:16,040 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T19:23:16,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T19:23:16,040 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T19:23:16,041 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T19:23:16,042 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T19:23:16,042 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731525796040Disabling compacts and flushes for region at 1731525796040Disabling writes for close at 1731525796040Writing region close event to WAL at 1731525796041 (+1 ms)Closed at 1731525796042 (+1 ms) 2024-11-13T19:23:16,046 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:23:16,046 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T19:23:16,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T19:23:16,059 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T19:23:16,062 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T19:23:16,219 DEBUG [e4b56bc8c4b3:44929 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-13T19:23:16,230 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:16,237 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e4b56bc8c4b3,40371,1731525792389, state=OPENING 2024-11-13T19:23:16,272 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T19:23:16,280 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:16,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:23:16,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:23:16,282 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:23:16,284 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T19:23:16,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:23:16,466 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T19:23:16,470 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54939, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T19:23:16,481 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T19:23:16,481 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T19:23:16,482 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T19:23:16,486 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4b56bc8c4b3%2C40371%2C1731525792389.meta, suffix=.meta, logDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/WALs/e4b56bc8c4b3,40371,1731525792389, archiveDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/oldWALs, maxLogs=32 2024-11-13T19:23:16,501 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/WALs/e4b56bc8c4b3,40371,1731525792389/e4b56bc8c4b3%2C40371%2C1731525792389.meta.1731525796488.meta, exclude list is [], retry=0 2024-11-13T19:23:16,505 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38613,DS-af2a52b2-01a0-47e1-95c9-9a760e7434ba,DISK] 2024-11-13T19:23:16,509 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/WALs/e4b56bc8c4b3,40371,1731525792389/e4b56bc8c4b3%2C40371%2C1731525792389.meta.1731525796488.meta 2024-11-13T19:23:16,509 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44019:44019)] 2024-11-13T19:23:16,510 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:23:16,512 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T19:23:16,515 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T19:23:16,521 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T19:23:16,526 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T19:23:16,526 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:16,527 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T19:23:16,527 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T19:23:16,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T19:23:16,533 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T19:23:16,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T19:23:16,537 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T19:23:16,537 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,538 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T19:23:16,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T19:23:16,540 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,541 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T19:23:16,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T19:23:16,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T19:23:16,544 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T19:23:16,545 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740 2024-11-13T19:23:16,548 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740 2024-11-13T19:23:16,551 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T19:23:16,551 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T19:23:16,552 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:23:16,556 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T19:23:16,557 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59451490, jitterRate=-0.114103764295578}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:23:16,558 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T19:23:16,560 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731525796528Writing region info on filesystem at 1731525796528Initializing all the Stores at 1731525796530 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731525796530Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731525796531 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525796531Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731525796531Cleaning up temporary data from old regions at 1731525796551 (+20 ms)Running coprocessor post-open hooks at 1731525796558 (+7 ms)Region opened successfully at 1731525796560 (+2 ms) 2024-11-13T19:23:16,566 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731525796459 2024-11-13T19:23:16,577 DEBUG [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T19:23:16,578 INFO [RS_OPEN_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T19:23:16,580 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:16,582 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e4b56bc8c4b3,40371,1731525792389, state=OPEN 2024-11-13T19:23:16,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:23:16,616 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T19:23:16,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:23:16,617 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T19:23:16,617 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:16,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T19:23:16,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e4b56bc8c4b3,40371,1731525792389 in 330 msec 2024-11-13T19:23:16,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T19:23:16,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 577 msec 2024-11-13T19:23:16,636 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T19:23:16,636 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T19:23:16,654 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:16,655 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:16,673 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:16,676 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42595, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:16,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6350 sec 2024-11-13T19:23:16,700 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731525796700, completionTime=-1 2024-11-13T19:23:16,703 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-13T19:23:16,703 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T19:23:16,726 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-13T19:23:16,726 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731525856726 2024-11-13T19:23:16,726 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731525916726 2024-11-13T19:23:16,726 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 23 msec 2024-11-13T19:23:16,728 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,44929,1731525791607-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:16,729 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,44929,1731525791607-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:16,729 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,44929,1731525791607-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:16,730 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e4b56bc8c4b3:44929, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:16,731 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:16,731 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:16,736 DEBUG [master/e4b56bc8c4b3:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T19:23:16,760 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.202sec 2024-11-13T19:23:16,761 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T19:23:16,762 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T19:23:16,763 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T19:23:16,764 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T19:23:16,764 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T19:23:16,765 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,44929,1731525791607-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T19:23:16,766 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,44929,1731525791607-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T19:23:16,773 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T19:23:16,774 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T19:23:16,774 INFO [master/e4b56bc8c4b3:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4b56bc8c4b3,44929,1731525791607-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T19:23:16,799 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:16,802 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T19:23:16,802 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T19:23:16,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:16,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:16,814 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@531238f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:16,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:16,825 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:16,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:16,828 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:16,835 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:16,838 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:16,858 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:16,871 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=240, ProcessCount=11, AvailableMemoryMB=1275 2024-11-13T19:23:16,879 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:23:16,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@680215db 2024-11-13T19:23:16,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:23:16,907 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:23:16,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:23:16,915 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:23:16,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:16,929 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:23:16,936 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-13T19:23:16,937 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:16,940 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:23:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:23:16,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741835_1011 (size=959) 2024-11-13T19:23:17,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:23:17,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:23:17,363 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:23:17,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741836_1012 (size=53) 2024-11-13T19:23:17,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:23:17,776 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:17,777 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1334c8616b794fd8c4fb42c4bfbc0d50, disabling compactions & flushes 2024-11-13T19:23:17,777 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:17,777 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:17,777 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. after waiting 0 ms 2024-11-13T19:23:17,777 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:17,777 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:17,778 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1334c8616b794fd8c4fb42c4bfbc0d50: Waiting for close lock at 1731525797776Disabling compacts and flushes for region at 1731525797776Disabling writes for close at 1731525797777 (+1 ms)Writing region close event to WAL at 1731525797777Closed at 1731525797777 2024-11-13T19:23:17,782 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:23:17,788 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731525797782"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731525797782"}]},"ts":"1731525797782"} 2024-11-13T19:23:17,797 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:23:17,799 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:23:17,801 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525797799"}]},"ts":"1731525797799"} 2024-11-13T19:23:17,805 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:23:17,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, ASSIGN}] 2024-11-13T19:23:17,809 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, ASSIGN 2024-11-13T19:23:17,811 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, ASSIGN; state=OFFLINE, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=false 2024-11-13T19:23:17,963 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1334c8616b794fd8c4fb42c4bfbc0d50, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:17,968 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, ASSIGN because future has completed 2024-11-13T19:23:17,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:23:18,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:23:18,131 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:18,131 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:23:18,132 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,132 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:18,132 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,132 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,135 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,138 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1334c8616b794fd8c4fb42c4bfbc0d50 columnFamilyName A 2024-11-13T19:23:18,139 DEBUG [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:18,140 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] regionserver.HStore(327): Store=1334c8616b794fd8c4fb42c4bfbc0d50/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:18,140 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,143 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1334c8616b794fd8c4fb42c4bfbc0d50 columnFamilyName B 2024-11-13T19:23:18,143 DEBUG [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:18,144 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] regionserver.HStore(327): Store=1334c8616b794fd8c4fb42c4bfbc0d50/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:18,144 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,147 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1334c8616b794fd8c4fb42c4bfbc0d50 columnFamilyName C 2024-11-13T19:23:18,147 DEBUG [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:18,148 INFO [StoreOpener-1334c8616b794fd8c4fb42c4bfbc0d50-1 {}] regionserver.HStore(327): Store=1334c8616b794fd8c4fb42c4bfbc0d50/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:18,148 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,149 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,150 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,152 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,152 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,153 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:23:18,155 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,159 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:23:18,160 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1334c8616b794fd8c4fb42c4bfbc0d50; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59705057, jitterRate=-0.11032532155513763}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:23:18,160 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:18,161 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1334c8616b794fd8c4fb42c4bfbc0d50: Running coprocessor pre-open hook at 1731525798133Writing region info on filesystem at 1731525798133Initializing all the Stores at 1731525798135 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525798135Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525798135Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525798135Cleaning up temporary data from old regions at 1731525798152 (+17 ms)Running coprocessor post-open hooks at 1731525798160 (+8 ms)Region opened successfully at 1731525798161 (+1 ms) 2024-11-13T19:23:18,163 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., pid=6, masterSystemTime=1731525798124 2024-11-13T19:23:18,167 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:18,167 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:18,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1334c8616b794fd8c4fb42c4bfbc0d50, regionState=OPEN, openSeqNum=2, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:18,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:23:18,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T19:23:18,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 in 205 msec 2024-11-13T19:23:18,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T19:23:18,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, ASSIGN in 372 msec 2024-11-13T19:23:18,186 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:23:18,187 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525798187"}]},"ts":"1731525798187"} 2024-11-13T19:23:18,191 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:23:18,192 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:23:18,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2740 sec 2024-11-13T19:23:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T19:23:19,097 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:19,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,105 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,106 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b1276ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,108 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,109 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@448c518f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,111 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,113 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,116 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57096, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,118 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:23:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:19,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@24a7aac4 2024-11-13T19:23:19,122 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:23:19,124 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:23:19,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,130 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61788bf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,132 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,133 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,134 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5da37544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,137 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbb78b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,140 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f953883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,147 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f8ac40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,150 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e49c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,152 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,152 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,152 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,154 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4c9506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6706ff39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,159 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,159 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,160 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c2308e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,163 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ee049f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,165 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,165 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,165 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,166 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60486904, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,168 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,170 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f58bfae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,173 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a574470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,175 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655f6768, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,179 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0ceab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,181 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,182 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dbd72e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:19,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:19,187 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:19,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306cf84f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:19,190 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:19,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8e4fc72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:19,194 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,195 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,196 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,197 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,198 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,198 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,201 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,203 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,208 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,216 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,239 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-13T19:23:19,251 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,251 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,254 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,255 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,256 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:19,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:23:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:19,258 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:19,259 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,260 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:19,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,262 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:19,264 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:19,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:23:19,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:19,330 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:19,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:23:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:23:19,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:23:19,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:19,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b200021f699d446db30889ef1c116215 is 50, key is test_row_0/A:col10/1731525799324/Put/seqid=0 2024-11-13T19:23:19,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525859498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525859496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525859509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525859512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525859515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741837_1013 (size=12001) 2024-11-13T19:23:19,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b200021f699d446db30889ef1c116215 2024-11-13T19:23:19,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:23:19,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:23:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:19,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525859662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525859663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525859663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525859664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525859672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/eaaadb1a698c46e59ed2530461f32856 is 50, key is test_row_0/B:col10/1731525799324/Put/seqid=0 2024-11-13T19:23:19,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741838_1014 (size=12001) 2024-11-13T19:23:19,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/eaaadb1a698c46e59ed2530461f32856 2024-11-13T19:23:19,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/36bdabf2e6674fb199b8e93334550586 is 50, key is test_row_0/C:col10/1731525799324/Put/seqid=0 2024-11-13T19:23:19,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:23:19,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741839_1015 (size=12001) 2024-11-13T19:23:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:19,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525859875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525859873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525859875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:23:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525859883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:19,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525859884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:19,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:19,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:23:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:19,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:19,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:23:20,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:20,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:20,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525860183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,185 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525860183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525860192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,197 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525860193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525860203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,230 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/36bdabf2e6674fb199b8e93334550586 2024-11-13T19:23:20,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b200021f699d446db30889ef1c116215 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b200021f699d446db30889ef1c116215 2024-11-13T19:23:20,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b200021f699d446db30889ef1c116215, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T19:23:20,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/eaaadb1a698c46e59ed2530461f32856 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/eaaadb1a698c46e59ed2530461f32856 2024-11-13T19:23:20,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/eaaadb1a698c46e59ed2530461f32856, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T19:23:20,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/36bdabf2e6674fb199b8e93334550586 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/36bdabf2e6674fb199b8e93334550586 2024-11-13T19:23:20,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/36bdabf2e6674fb199b8e93334550586, entries=150, sequenceid=13, filesize=11.7 K 2024-11-13T19:23:20,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:23:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:20,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 977ms, sequenceid=13, compaction requested=false 2024-11-13T19:23:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:20,315 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,316 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T19:23:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:20,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:23:20,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-13T19:23:20,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:20,472 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:23:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b3ec2625cf604580ab6e7fcc269eab24 is 50, key is test_row_0/A:col10/1731525799508/Put/seqid=0 2024-11-13T19:23:20,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741840_1016 (size=12001) 2024-11-13T19:23:20,539 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b3ec2625cf604580ab6e7fcc269eab24 2024-11-13T19:23:20,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/00bcec48e0ad425d8021967d10547a13 is 50, key is test_row_0/B:col10/1731525799508/Put/seqid=0 2024-11-13T19:23:20,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741841_1017 (size=12001) 2024-11-13T19:23:20,575 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/00bcec48e0ad425d8021967d10547a13 2024-11-13T19:23:20,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/167c559a5adb47c9ad1b28e00bcdbaf0 is 50, key is test_row_0/C:col10/1731525799508/Put/seqid=0 2024-11-13T19:23:20,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741842_1018 (size=12001) 2024-11-13T19:23:20,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:20,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:20,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525860733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525860737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525860741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525860746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525860749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525860852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525860862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525860872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525860874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:20,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:20,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525860874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:20,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,044 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/167c559a5adb47c9ad1b28e00bcdbaf0 2024-11-13T19:23:21,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b3ec2625cf604580ab6e7fcc269eab24 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3ec2625cf604580ab6e7fcc269eab24 2024-11-13T19:23:21,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525861063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525861081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,097 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3ec2625cf604580ab6e7fcc269eab24, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:23:21,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/00bcec48e0ad425d8021967d10547a13 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/00bcec48e0ad425d8021967d10547a13 2024-11-13T19:23:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525861093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,115 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/00bcec48e0ad425d8021967d10547a13, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:23:21,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525861093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525861102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/167c559a5adb47c9ad1b28e00bcdbaf0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/167c559a5adb47c9ad1b28e00bcdbaf0 2024-11-13T19:23:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,136 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/167c559a5adb47c9ad1b28e00bcdbaf0, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:23:21,139 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 667ms, sequenceid=37, compaction requested=false 2024-11-13T19:23:21,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:21,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:21,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-13T19:23:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-13T19:23:21,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T19:23:21,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8830 sec 2024-11-13T19:23:21,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9260 sec 2024-11-13T19:23:21,236 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T19:23:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:21,394 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:21,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T19:23:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-13T19:23:21,403 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:21,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-13T19:23:21,408 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:23:21,410 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:21,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:21,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/6f1828abbcd847a195cfd8acfd6d7967 is 50, key is test_row_0/A:col10/1731525800724/Put/seqid=0 2024-11-13T19:23:21,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741843_1019 (size=14341) 2024-11-13T19:23:21,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/6f1828abbcd847a195cfd8acfd6d7967 2024-11-13T19:23:21,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/d7c5a74a7a004acfb8a0be7e3f16e41d is 50, key is test_row_0/B:col10/1731525800724/Put/seqid=0 2024-11-13T19:23:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525861467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525861471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525861476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525861483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525861485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,493 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:23:21,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741844_1020 (size=12001) 2024-11-13T19:23:21,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/d7c5a74a7a004acfb8a0be7e3f16e41d 2024-11-13T19:23:21,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5d1b1c1bd61142a88172308f57e5f087 is 50, key is test_row_0/C:col10/1731525800724/Put/seqid=0 2024-11-13T19:23:21,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741845_1021 (size=12001) 2024-11-13T19:23:21,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5d1b1c1bd61142a88172308f57e5f087 2024-11-13T19:23:21,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:23:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:21,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:21,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:21,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525861593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525861595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525861594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525861595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525861602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/6f1828abbcd847a195cfd8acfd6d7967 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/6f1828abbcd847a195cfd8acfd6d7967 2024-11-13T19:23:21,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/6f1828abbcd847a195cfd8acfd6d7967, entries=200, sequenceid=51, filesize=14.0 K 2024-11-13T19:23:21,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/d7c5a74a7a004acfb8a0be7e3f16e41d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/d7c5a74a7a004acfb8a0be7e3f16e41d 2024-11-13T19:23:21,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/d7c5a74a7a004acfb8a0be7e3f16e41d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T19:23:21,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5d1b1c1bd61142a88172308f57e5f087 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5d1b1c1bd61142a88172308f57e5f087 2024-11-13T19:23:21,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5d1b1c1bd61142a88172308f57e5f087, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T19:23:21,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 274ms, sequenceid=51, compaction requested=true 2024-11-13T19:23:21,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:21,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:21,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:21,678 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:21,682 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:21,683 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:21,685 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:21,686 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:21,687 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/eaaadb1a698c46e59ed2530461f32856, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/00bcec48e0ad425d8021967d10547a13, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/d7c5a74a7a004acfb8a0be7e3f16e41d] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=35.2 K 2024-11-13T19:23:21,688 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting eaaadb1a698c46e59ed2530461f32856, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731525799284 2024-11-13T19:23:21,689 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:21,690 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 00bcec48e0ad425d8021967d10547a13, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525799478 2024-11-13T19:23:21,690 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:21,690 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:21,690 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b200021f699d446db30889ef1c116215, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3ec2625cf604580ab6e7fcc269eab24, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/6f1828abbcd847a195cfd8acfd6d7967] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=37.4 K 2024-11-13T19:23:21,690 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c5a74a7a004acfb8a0be7e3f16e41d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525800724 2024-11-13T19:23:21,692 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b200021f699d446db30889ef1c116215, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731525799284 2024-11-13T19:23:21,693 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ec2625cf604580ab6e7fcc269eab24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525799478 2024-11-13T19:23:21,695 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f1828abbcd847a195cfd8acfd6d7967, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525800724 2024-11-13T19:23:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:23:21,732 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#9 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:21,733 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/1e8feef1d1ab4131a9fbe5c33beb65b8 is 50, key is test_row_0/A:col10/1731525800724/Put/seqid=0 2024-11-13T19:23:21,743 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-13T19:23:21,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:21,744 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T19:23:21,747 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:21,748 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/02f607160d0847b283c3770f107629b3 is 50, key is test_row_0/B:col10/1731525800724/Put/seqid=0 2024-11-13T19:23:21,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741846_1022 (size=12104) 2024-11-13T19:23:21,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/8720f776b6464cb180da5e50a5fb7211 is 50, key is test_row_0/A:col10/1731525801471/Put/seqid=0 2024-11-13T19:23:21,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741847_1023 (size=12104) 2024-11-13T19:23:21,819 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:21,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:21,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741848_1024 (size=12001) 2024-11-13T19:23:21,821 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/02f607160d0847b283c3770f107629b3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/02f607160d0847b283c3770f107629b3 2024-11-13T19:23:21,852 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into 02f607160d0847b283c3770f107629b3(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:21,852 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:21,858 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525801676; duration=0sec 2024-11-13T19:23:21,862 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:21,862 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:21,864 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:21,874 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:21,874 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:21,874 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:21,874 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/36bdabf2e6674fb199b8e93334550586, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/167c559a5adb47c9ad1b28e00bcdbaf0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5d1b1c1bd61142a88172308f57e5f087] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=35.2 K 2024-11-13T19:23:21,875 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 36bdabf2e6674fb199b8e93334550586, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731525799284 2024-11-13T19:23:21,876 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 167c559a5adb47c9ad1b28e00bcdbaf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525799478 2024-11-13T19:23:21,877 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1b1c1bd61142a88172308f57e5f087, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525800724 2024-11-13T19:23:21,906 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#12 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:21,907 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/b0cf850e7d54477b9990e1ff649693c4 is 50, key is test_row_0/C:col10/1731525800724/Put/seqid=0 2024-11-13T19:23:21,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741849_1025 (size=12104) 2024-11-13T19:23:21,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525861861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525861866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,940 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525861938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525861938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:21,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:21,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525861941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:21,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:23:22,045 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:23:22,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525862044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525862044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525862062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525862072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525862073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,077 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,131 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T19:23:22,132 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T19:23:22,135 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:23:22,135 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T19:23:22,136 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T19:23:22,136 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T19:23:22,138 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:23:22,138 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T19:23:22,199 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/1e8feef1d1ab4131a9fbe5c33beb65b8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/1e8feef1d1ab4131a9fbe5c33beb65b8 2024-11-13T19:23:22,216 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into 1e8feef1d1ab4131a9fbe5c33beb65b8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:22,216 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:22,216 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525801670; duration=0sec 2024-11-13T19:23:22,217 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:22,217 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:22,222 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/8720f776b6464cb180da5e50a5fb7211 2024-11-13T19:23:22,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/c82eac95641544029fd56d278ed407a1 is 50, key is test_row_0/B:col10/1731525801471/Put/seqid=0 2024-11-13T19:23:22,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525862252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525862262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741850_1026 (size=12001) 2024-11-13T19:23:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525862282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525862282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525862283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,292 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/c82eac95641544029fd56d278ed407a1 2024-11-13T19:23:22,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/07211a5a88864e3bb49722652a13aafa is 50, key is test_row_0/C:col10/1731525801471/Put/seqid=0 2024-11-13T19:23:22,344 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/b0cf850e7d54477b9990e1ff649693c4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b0cf850e7d54477b9990e1ff649693c4 2024-11-13T19:23:22,360 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into b0cf850e7d54477b9990e1ff649693c4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:22,360 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:22,360 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525801680; duration=0sec 2024-11-13T19:23:22,360 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:22,360 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:22,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741851_1027 (size=12001) 2024-11-13T19:23:22,362 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/07211a5a88864e3bb49722652a13aafa 2024-11-13T19:23:22,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/8720f776b6464cb180da5e50a5fb7211 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8720f776b6464cb180da5e50a5fb7211 2024-11-13T19:23:22,402 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8720f776b6464cb180da5e50a5fb7211, entries=150, sequenceid=73, filesize=11.7 K 2024-11-13T19:23:22,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/c82eac95641544029fd56d278ed407a1 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/c82eac95641544029fd56d278ed407a1 2024-11-13T19:23:22,414 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/c82eac95641544029fd56d278ed407a1, entries=150, sequenceid=73, filesize=11.7 K 2024-11-13T19:23:22,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/07211a5a88864e3bb49722652a13aafa as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/07211a5a88864e3bb49722652a13aafa 2024-11-13T19:23:22,432 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/07211a5a88864e3bb49722652a13aafa, entries=150, sequenceid=73, filesize=11.7 K 2024-11-13T19:23:22,438 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=87.22 KB/89310 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 694ms, sequenceid=73, compaction requested=false 2024-11-13T19:23:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:22,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-13T19:23:22,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-13T19:23:22,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-13T19:23:22,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0320 sec 2024-11-13T19:23:22,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0470 sec 2024-11-13T19:23:22,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-13T19:23:22,542 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:22,542 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:22,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-13T19:23:22,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:23:22,546 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:22,548 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:22,548 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:22,581 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:22,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T19:23:22,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:22,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/493b0634adff4959b7458c38a1006dc8 is 50, key is test_row_0/A:col10/1731525802576/Put/seqid=0 2024-11-13T19:23:22,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741852_1028 (size=12001) 2024-11-13T19:23:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525862633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525862630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525862643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525862643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525862643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:23:22,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:23:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:22,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:22,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:22,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:22,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:22,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525862751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525862752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525862752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525862752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525862761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:23:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:22,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:22,860 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:22,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:23:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525862962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525862962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525862962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525862962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:22,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525862972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:22,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,015 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:23:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/493b0634adff4959b7458c38a1006dc8 2024-11-13T19:23:23,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f241b5121987497d93ebc315a7f5203f is 50, key is test_row_0/B:col10/1731525802576/Put/seqid=0 2024-11-13T19:23:23,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741853_1029 (size=12001) 2024-11-13T19:23:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:23:23,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:23:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:23,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525863281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525863281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525863281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525863282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525863282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:23:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f241b5121987497d93ebc315a7f5203f 2024-11-13T19:23:23,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/21178898cfc8427eba1b3aa56387ee8b is 50, key is test_row_0/C:col10/1731525802576/Put/seqid=0 2024-11-13T19:23:23,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:23:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:23,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741854_1030 (size=12001) 2024-11-13T19:23:23,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/21178898cfc8427eba1b3aa56387ee8b 2024-11-13T19:23:23,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/493b0634adff4959b7458c38a1006dc8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/493b0634adff4959b7458c38a1006dc8 2024-11-13T19:23:23,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/493b0634adff4959b7458c38a1006dc8, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T19:23:23,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f241b5121987497d93ebc315a7f5203f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f241b5121987497d93ebc315a7f5203f 2024-11-13T19:23:23,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f241b5121987497d93ebc315a7f5203f, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T19:23:23,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/21178898cfc8427eba1b3aa56387ee8b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/21178898cfc8427eba1b3aa56387ee8b 2024-11-13T19:23:23,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/21178898cfc8427eba1b3aa56387ee8b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T19:23:23,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=114.05 KB/116790 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 992ms, sequenceid=94, compaction requested=true 2024-11-13T19:23:23,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:23,574 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:23,574 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:23,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:23,576 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:23,576 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:23,576 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,576 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/1e8feef1d1ab4131a9fbe5c33beb65b8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8720f776b6464cb180da5e50a5fb7211, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/493b0634adff4959b7458c38a1006dc8] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=35.3 K 2024-11-13T19:23:23,577 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:23,577 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:23,577 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,577 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/02f607160d0847b283c3770f107629b3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/c82eac95641544029fd56d278ed407a1, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f241b5121987497d93ebc315a7f5203f] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=35.3 K 2024-11-13T19:23:23,578 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 02f607160d0847b283c3770f107629b3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525800724 2024-11-13T19:23:23,578 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8feef1d1ab4131a9fbe5c33beb65b8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525800724 2024-11-13T19:23:23,579 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8720f776b6464cb180da5e50a5fb7211, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731525801471 2024-11-13T19:23:23,579 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c82eac95641544029fd56d278ed407a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731525801471 2024-11-13T19:23:23,579 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f241b5121987497d93ebc315a7f5203f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525801901 2024-11-13T19:23:23,579 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 493b0634adff4959b7458c38a1006dc8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525801901 2024-11-13T19:23:23,605 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#18 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:23,605 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b3e090fa86394f5d878609c5cff3a808 is 50, key is test_row_0/A:col10/1731525802576/Put/seqid=0 2024-11-13T19:23:23,612 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:23,613 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/a558395291fb4261a4a0d3deb7b70ccf is 50, key is test_row_0/B:col10/1731525802576/Put/seqid=0 2024-11-13T19:23:23,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-13T19:23:23,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,655 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T19:23:23,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/02885f2f61094b06967cb7276ec4ff6b is 50, key is test_row_0/A:col10/1731525802635/Put/seqid=0 2024-11-13T19:23:23,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741855_1031 (size=12207) 2024-11-13T19:23:23,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741856_1032 (size=12207) 2024-11-13T19:23:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:23:23,690 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b3e090fa86394f5d878609c5cff3a808 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3e090fa86394f5d878609c5cff3a808 2024-11-13T19:23:23,692 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/a558395291fb4261a4a0d3deb7b70ccf as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a558395291fb4261a4a0d3deb7b70ccf 2024-11-13T19:23:23,702 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into b3e090fa86394f5d878609c5cff3a808(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:23,703 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:23,703 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525803573; duration=0sec 2024-11-13T19:23:23,704 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:23,704 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:23,704 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:23,706 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:23,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741857_1033 (size=12001) 2024-11-13T19:23:23,706 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:23,708 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:23,708 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b0cf850e7d54477b9990e1ff649693c4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/07211a5a88864e3bb49722652a13aafa, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/21178898cfc8427eba1b3aa56387ee8b] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=35.3 K 2024-11-13T19:23:23,709 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0cf850e7d54477b9990e1ff649693c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525800724 2024-11-13T19:23:23,712 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into a558395291fb4261a4a0d3deb7b70ccf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:23,712 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07211a5a88864e3bb49722652a13aafa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731525801471 2024-11-13T19:23:23,712 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:23,712 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525803574; duration=0sec 2024-11-13T19:23:23,712 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:23,712 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:23,712 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21178898cfc8427eba1b3aa56387ee8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525801901 2024-11-13T19:23:23,730 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#21 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:23,730 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/bfebe18b7e9147ffb9cfc258a96ed8af is 50, key is test_row_0/C:col10/1731525802576/Put/seqid=0 2024-11-13T19:23:23,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741858_1034 (size=12207) 2024-11-13T19:23:23,783 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/bfebe18b7e9147ffb9cfc258a96ed8af as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bfebe18b7e9147ffb9cfc258a96ed8af 2024-11-13T19:23:23,795 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into bfebe18b7e9147ffb9cfc258a96ed8af(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:23,795 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:23,795 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525803574; duration=0sec 2024-11-13T19:23:23,796 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:23,796 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:23,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525863839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525863838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525863844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525863846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525863848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525863961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525863962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525863962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525863963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:23,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525863965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,974 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:23,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,104 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/02885f2f61094b06967cb7276ec4ff6b 2024-11-13T19:23:24,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/caa09aab9cb2408caff5354022c91e6e is 50, key is test_row_0/B:col10/1731525802635/Put/seqid=0 2024-11-13T19:23:24,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741859_1035 (size=12001) 2024-11-13T19:23:24,156 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/caa09aab9cb2408caff5354022c91e6e 2024-11-13T19:23:24,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/25eacbb5bd4749459a3ed836dbf79679 is 50, key is test_row_0/C:col10/1731525802635/Put/seqid=0 2024-11-13T19:23:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525864182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525864182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525864182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525864183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525864186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741860_1036 (size=12001) 2024-11-13T19:23:24,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525864492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525864492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,494 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525864492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525864492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525864492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:24,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:24,594 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/25eacbb5bd4749459a3ed836dbf79679 2024-11-13T19:23:24,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/02885f2f61094b06967cb7276ec4ff6b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/02885f2f61094b06967cb7276ec4ff6b 2024-11-13T19:23:24,629 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/02885f2f61094b06967cb7276ec4ff6b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:23:24,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/caa09aab9cb2408caff5354022c91e6e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/caa09aab9cb2408caff5354022c91e6e 2024-11-13T19:23:24,645 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/caa09aab9cb2408caff5354022c91e6e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:23:24,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/25eacbb5bd4749459a3ed836dbf79679 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/25eacbb5bd4749459a3ed836dbf79679 2024-11-13T19:23:24,660 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/25eacbb5bd4749459a3ed836dbf79679, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:23:24,662 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 1006ms, sequenceid=114, compaction requested=false 2024-11-13T19:23:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:24,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-13T19:23:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-13T19:23:24,670 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-13T19:23:24,671 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1180 sec 2024-11-13T19:23:24,675 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.1280 sec 2024-11-13T19:23:24,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-13T19:23:24,702 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:24,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-13T19:23:24,707 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:23:24,709 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:24,710 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:23:24,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-13T19:23:24,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:24,865 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T19:23:24,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b7bfd88eef3a4defada130cb9df7accb is 50, key is test_row_0/A:col10/1731525803842/Put/seqid=0 2024-11-13T19:23:24,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741861_1037 (size=12151) 2024-11-13T19:23:24,909 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b7bfd88eef3a4defada130cb9df7accb 2024-11-13T19:23:24,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/309387167d434fcfa97ae1711a92b1ee is 50, key is test_row_0/B:col10/1731525803842/Put/seqid=0 2024-11-13T19:23:24,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741862_1038 (size=12151) 2024-11-13T19:23:24,948 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/309387167d434fcfa97ae1711a92b1ee 2024-11-13T19:23:24,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/fdd1a76df0b4417dbdb8ab78bf4feeb9 is 50, key is test_row_0/C:col10/1731525803842/Put/seqid=0 2024-11-13T19:23:25,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741863_1039 (size=12151) 2024-11-13T19:23:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:25,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:23:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525865058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525865058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,069 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525865065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525865068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525865068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525865172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525865172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525865181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525865181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525865181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:23:25,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525865381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525865381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525865392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525865392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525865392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,406 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/fdd1a76df0b4417dbdb8ab78bf4feeb9 2024-11-13T19:23:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b7bfd88eef3a4defada130cb9df7accb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b7bfd88eef3a4defada130cb9df7accb 2024-11-13T19:23:25,426 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b7bfd88eef3a4defada130cb9df7accb, entries=150, sequenceid=133, filesize=11.9 K 2024-11-13T19:23:25,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/309387167d434fcfa97ae1711a92b1ee as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/309387167d434fcfa97ae1711a92b1ee 2024-11-13T19:23:25,442 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/309387167d434fcfa97ae1711a92b1ee, entries=150, sequenceid=133, filesize=11.9 K 2024-11-13T19:23:25,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/fdd1a76df0b4417dbdb8ab78bf4feeb9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/fdd1a76df0b4417dbdb8ab78bf4feeb9 2024-11-13T19:23:25,457 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/fdd1a76df0b4417dbdb8ab78bf4feeb9, entries=150, sequenceid=133, filesize=11.9 K 2024-11-13T19:23:25,459 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 593ms, sequenceid=133, compaction requested=true 2024-11-13T19:23:25,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:25,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:25,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-13T19:23:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-13T19:23:25,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T19:23:25,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 751 msec 2024-11-13T19:23:25,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 764 msec 2024-11-13T19:23:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:25,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:25,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T19:23:25,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/473756de85764cf2a8bcd260042069bb is 50, key is test_row_0/A:col10/1731525805040/Put/seqid=0 2024-11-13T19:23:25,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525865713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741864_1040 (size=16931) 2024-11-13T19:23:25,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/473756de85764cf2a8bcd260042069bb 2024-11-13T19:23:25,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525865716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525865718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525865720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525865722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,728 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,729 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/4987763020914de89274ba6aaaa69ec3 is 50, key is test_row_0/B:col10/1731525805040/Put/seqid=0 2024-11-13T19:23:25,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741865_1041 (size=12151) 2024-11-13T19:23:25,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/4987763020914de89274ba6aaaa69ec3 2024-11-13T19:23:25,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a23cc797ab98410ab0e56c6091d1d66e is 50, key is test_row_0/C:col10/1731525805040/Put/seqid=0 2024-11-13T19:23:25,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525865832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525865833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525865833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525865833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525865834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:25,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:25,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-13T19:23:25,841 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:25,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-13T19:23:25,848 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:25,850 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:25,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:23:25,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741866_1042 (size=12151) 2024-11-13T19:23:25,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a23cc797ab98410ab0e56c6091d1d66e 2024-11-13T19:23:25,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/473756de85764cf2a8bcd260042069bb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/473756de85764cf2a8bcd260042069bb 2024-11-13T19:23:25,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/473756de85764cf2a8bcd260042069bb, entries=250, sequenceid=156, filesize=16.5 K 2024-11-13T19:23:25,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/4987763020914de89274ba6aaaa69ec3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/4987763020914de89274ba6aaaa69ec3 2024-11-13T19:23:25,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/4987763020914de89274ba6aaaa69ec3, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T19:23:25,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a23cc797ab98410ab0e56c6091d1d66e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a23cc797ab98410ab0e56c6091d1d66e 2024-11-13T19:23:25,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a23cc797ab98410ab0e56c6091d1d66e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T19:23:25,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 217ms, sequenceid=156, compaction requested=true 2024-11-13T19:23:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:25,913 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:25,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:25,913 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:25,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:25,915 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:25,916 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:25,916 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:25,916 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3e090fa86394f5d878609c5cff3a808, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/02885f2f61094b06967cb7276ec4ff6b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b7bfd88eef3a4defada130cb9df7accb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/473756de85764cf2a8bcd260042069bb] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=52.0 K 2024-11-13T19:23:25,917 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3e090fa86394f5d878609c5cff3a808, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525801901 2024-11-13T19:23:25,917 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:25,917 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:25,917 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:25,917 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a558395291fb4261a4a0d3deb7b70ccf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/caa09aab9cb2408caff5354022c91e6e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/309387167d434fcfa97ae1711a92b1ee, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/4987763020914de89274ba6aaaa69ec3] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=47.4 K 2024-11-13T19:23:25,918 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a558395291fb4261a4a0d3deb7b70ccf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525801901 2024-11-13T19:23:25,918 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02885f2f61094b06967cb7276ec4ff6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525802635 2024-11-13T19:23:25,919 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting caa09aab9cb2408caff5354022c91e6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525802635 2024-11-13T19:23:25,919 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 309387167d434fcfa97ae1711a92b1ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731525803834 2024-11-13T19:23:25,920 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4987763020914de89274ba6aaaa69ec3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525805040 2024-11-13T19:23:25,920 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7bfd88eef3a4defada130cb9df7accb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731525803834 2024-11-13T19:23:25,923 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 473756de85764cf2a8bcd260042069bb, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525805040 2024-11-13T19:23:25,957 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:25,957 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:25,958 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/a9ef1c2f0f6c4970bcf0b2e8883f892e is 50, key is test_row_0/B:col10/1731525805040/Put/seqid=0 2024-11-13T19:23:25,959 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/12f0b28564734dd18283b54bfa847e43 is 50, key is test_row_0/A:col10/1731525805040/Put/seqid=0 2024-11-13T19:23:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:23:25,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741867_1043 (size=12493) 2024-11-13T19:23:25,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741868_1044 (size=12493) 2024-11-13T19:23:26,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-13T19:23:26,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:26,005 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T19:23:26,014 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/a9ef1c2f0f6c4970bcf0b2e8883f892e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a9ef1c2f0f6c4970bcf0b2e8883f892e 2024-11-13T19:23:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/68cf528a00f44cc9a4039eb9617387be is 50, key is test_row_0/A:col10/1731525805717/Put/seqid=0 2024-11-13T19:23:26,026 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into a9ef1c2f0f6c4970bcf0b2e8883f892e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:26,026 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:26,026 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=12, startTime=1731525805913; duration=0sec 2024-11-13T19:23:26,027 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:26,027 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:26,027 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:26,034 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:26,034 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:26,034 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:26,035 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bfebe18b7e9147ffb9cfc258a96ed8af, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/25eacbb5bd4749459a3ed836dbf79679, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/fdd1a76df0b4417dbdb8ab78bf4feeb9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a23cc797ab98410ab0e56c6091d1d66e] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=47.4 K 2024-11-13T19:23:26,036 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting bfebe18b7e9147ffb9cfc258a96ed8af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525801901 2024-11-13T19:23:26,036 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 25eacbb5bd4749459a3ed836dbf79679, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525802635 2024-11-13T19:23:26,039 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting fdd1a76df0b4417dbdb8ab78bf4feeb9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731525803834 2024-11-13T19:23:26,040 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a23cc797ab98410ab0e56c6091d1d66e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525805040 2024-11-13T19:23:26,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741869_1045 (size=12151) 2024-11-13T19:23:26,046 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/68cf528a00f44cc9a4039eb9617387be 2024-11-13T19:23:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:26,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:26,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/07d96ab1ae9e46ec9da0b261694884b4 is 50, key is test_row_0/B:col10/1731525805717/Put/seqid=0 2024-11-13T19:23:26,090 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#34 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:26,092 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/970dde99b4d74ae0b96aae25cd141eb7 is 50, key is test_row_0/C:col10/1731525805040/Put/seqid=0 2024-11-13T19:23:26,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525866096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525866098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741870_1046 (size=12151) 2024-11-13T19:23:26,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525866101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525866101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525866103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,127 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/07d96ab1ae9e46ec9da0b261694884b4 2024-11-13T19:23:26,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741871_1047 (size=12493) 2024-11-13T19:23:26,150 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/970dde99b4d74ae0b96aae25cd141eb7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/970dde99b4d74ae0b96aae25cd141eb7 2024-11-13T19:23:26,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/4aa53fefd93a4eaabffbe0344ff3e866 is 50, key is test_row_0/C:col10/1731525805717/Put/seqid=0 2024-11-13T19:23:26,162 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into 970dde99b4d74ae0b96aae25cd141eb7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:26,162 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:26,162 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=12, startTime=1731525805914; duration=0sec 2024-11-13T19:23:26,162 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:26,162 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:23:26,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741872_1048 (size=12151) 2024-11-13T19:23:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525866211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525866212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525866222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,224 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525866222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525866222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,407 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/12f0b28564734dd18283b54bfa847e43 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/12f0b28564734dd18283b54bfa847e43 2024-11-13T19:23:26,420 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into 12f0b28564734dd18283b54bfa847e43(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:26,420 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:26,420 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=12, startTime=1731525805913; duration=0sec 2024-11-13T19:23:26,421 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:26,421 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:26,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525866422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525866422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,425 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525866431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,434 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525866432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525866432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:23:26,609 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/4aa53fefd93a4eaabffbe0344ff3e866 2024-11-13T19:23:26,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/68cf528a00f44cc9a4039eb9617387be as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/68cf528a00f44cc9a4039eb9617387be 2024-11-13T19:23:26,625 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/68cf528a00f44cc9a4039eb9617387be, entries=150, sequenceid=169, filesize=11.9 K 2024-11-13T19:23:26,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/07d96ab1ae9e46ec9da0b261694884b4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/07d96ab1ae9e46ec9da0b261694884b4 2024-11-13T19:23:26,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/07d96ab1ae9e46ec9da0b261694884b4, entries=150, sequenceid=169, filesize=11.9 K 2024-11-13T19:23:26,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/4aa53fefd93a4eaabffbe0344ff3e866 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/4aa53fefd93a4eaabffbe0344ff3e866 2024-11-13T19:23:26,662 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/4aa53fefd93a4eaabffbe0344ff3e866, entries=150, sequenceid=169, filesize=11.9 K 2024-11-13T19:23:26,664 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 659ms, sequenceid=169, compaction requested=false 2024-11-13T19:23:26,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:26,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:26,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-13T19:23:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-13T19:23:26,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-13T19:23:26,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 816 msec 2024-11-13T19:23:26,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 830 msec 2024-11-13T19:23:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:26,734 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:26,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T19:23:26,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/4a8c45cd38b748e4835154fb3e19629f is 50, key is test_row_0/A:col10/1731525806097/Put/seqid=0 2024-11-13T19:23:26,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525866745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525866746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525866750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525866750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525866750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741873_1049 (size=14541) 2024-11-13T19:23:26,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/4a8c45cd38b748e4835154fb3e19629f 2024-11-13T19:23:26,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e0e44c96c77348c3ae2fe2b83ce6ef37 is 50, key is test_row_0/B:col10/1731525806097/Put/seqid=0 2024-11-13T19:23:26,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741874_1050 (size=12151) 2024-11-13T19:23:26,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e0e44c96c77348c3ae2fe2b83ce6ef37 2024-11-13T19:23:26,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/6fe5dc32a8aa46b29f061bf22240c1b5 is 50, key is test_row_0/C:col10/1731525806097/Put/seqid=0 2024-11-13T19:23:26,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741875_1051 (size=12151) 2024-11-13T19:23:26,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/6fe5dc32a8aa46b29f061bf22240c1b5 2024-11-13T19:23:26,845 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/4a8c45cd38b748e4835154fb3e19629f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/4a8c45cd38b748e4835154fb3e19629f 2024-11-13T19:23:26,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/4a8c45cd38b748e4835154fb3e19629f, entries=200, sequenceid=199, filesize=14.2 K 2024-11-13T19:23:26,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e0e44c96c77348c3ae2fe2b83ce6ef37 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e0e44c96c77348c3ae2fe2b83ce6ef37 2024-11-13T19:23:26,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525866862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525866862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525866862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525866863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525866862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:26,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e0e44c96c77348c3ae2fe2b83ce6ef37, entries=150, sequenceid=199, filesize=11.9 K 2024-11-13T19:23:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:26,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:26,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/6fe5dc32a8aa46b29f061bf22240c1b5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6fe5dc32a8aa46b29f061bf22240c1b5 2024-11-13T19:23:26,884 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6fe5dc32a8aa46b29f061bf22240c1b5, entries=150, sequenceid=199, filesize=11.9 K 2024-11-13T19:23:26,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 152ms, sequenceid=199, compaction requested=true 2024-11-13T19:23:26,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:26,888 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:26,888 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:26,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:26,890 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:26,891 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:26,891 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:26,891 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/12f0b28564734dd18283b54bfa847e43, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/68cf528a00f44cc9a4039eb9617387be, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/4a8c45cd38b748e4835154fb3e19629f] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=38.3 K 2024-11-13T19:23:26,892 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:26,892 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:26,892 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:26,892 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a9ef1c2f0f6c4970bcf0b2e8883f892e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/07d96ab1ae9e46ec9da0b261694884b4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e0e44c96c77348c3ae2fe2b83ce6ef37] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=35.9 K 2024-11-13T19:23:26,893 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12f0b28564734dd18283b54bfa847e43, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525805040 2024-11-13T19:23:26,893 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ef1c2f0f6c4970bcf0b2e8883f892e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525805040 2024-11-13T19:23:26,894 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 07d96ab1ae9e46ec9da0b261694884b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731525805715 2024-11-13T19:23:26,894 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68cf528a00f44cc9a4039eb9617387be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731525805715 2024-11-13T19:23:26,895 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a8c45cd38b748e4835154fb3e19629f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731525806097 2024-11-13T19:23:26,895 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e0e44c96c77348c3ae2fe2b83ce6ef37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731525806097 2024-11-13T19:23:26,917 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:26,917 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/147b8e70b3d344898e74ff323c42d0f7 is 50, key is test_row_0/B:col10/1731525806097/Put/seqid=0 2024-11-13T19:23:26,918 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:26,918 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/ede83bb5010f41c3866e93415c08b893 is 50, key is test_row_0/A:col10/1731525806097/Put/seqid=0 2024-11-13T19:23:26,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741876_1052 (size=12595) 2024-11-13T19:23:26,945 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/147b8e70b3d344898e74ff323c42d0f7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/147b8e70b3d344898e74ff323c42d0f7 2024-11-13T19:23:26,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741877_1053 (size=12595) 2024-11-13T19:23:26,955 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into 147b8e70b3d344898e74ff323c42d0f7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:26,955 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:26,955 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525806888; duration=0sec 2024-11-13T19:23:26,956 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:26,956 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:26,956 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:26,958 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:26,958 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:26,959 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:26,959 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/970dde99b4d74ae0b96aae25cd141eb7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/4aa53fefd93a4eaabffbe0344ff3e866, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6fe5dc32a8aa46b29f061bf22240c1b5] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=35.9 K 2024-11-13T19:23:26,959 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 970dde99b4d74ae0b96aae25cd141eb7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525805040 2024-11-13T19:23:26,961 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aa53fefd93a4eaabffbe0344ff3e866, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731525805715 2024-11-13T19:23:26,962 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fe5dc32a8aa46b29f061bf22240c1b5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731525806097 2024-11-13T19:23:26,971 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/ede83bb5010f41c3866e93415c08b893 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/ede83bb5010f41c3866e93415c08b893 2024-11-13T19:23:26,982 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into ede83bb5010f41c3866e93415c08b893(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:26,983 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:26,983 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525806888; duration=0sec 2024-11-13T19:23:26,983 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:26,983 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:26,990 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#41 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:26,990 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/b5d6e2589086416893a7123681308e7b is 50, key is test_row_0/C:col10/1731525806097/Put/seqid=0 2024-11-13T19:23:26,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-13T19:23:26,992 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:26,993 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-13T19:23:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:23:26,995 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:26,997 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:26,997 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:27,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741878_1054 (size=12595) 2024-11-13T19:23:27,021 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/b5d6e2589086416893a7123681308e7b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b5d6e2589086416893a7123681308e7b 2024-11-13T19:23:27,033 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into b5d6e2589086416893a7123681308e7b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:27,033 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:27,033 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525806888; duration=0sec 2024-11-13T19:23:27,034 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:27,034 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:27,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:27,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:27,075 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:23:27,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/607bf08ede824569bc6c39e4dcc87f7c is 50, key is test_row_0/A:col10/1731525806747/Put/seqid=0 2024-11-13T19:23:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:23:27,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741879_1055 (size=16931) 2024-11-13T19:23:27,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/607bf08ede824569bc6c39e4dcc87f7c 2024-11-13T19:23:27,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525867120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525867133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525867134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525867138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525867136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:23:27,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:27,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:27,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:27,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:27,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5288daab5ef14f3f81471f1fca863c66 is 50, key is test_row_0/B:col10/1731525806747/Put/seqid=0 2024-11-13T19:23:27,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741880_1056 (size=12151) 2024-11-13T19:23:27,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5288daab5ef14f3f81471f1fca863c66 2024-11-13T19:23:27,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/e6353d4687824eaa91ca3f30332065e9 is 50, key is test_row_0/C:col10/1731525806747/Put/seqid=0 2024-11-13T19:23:27,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741881_1057 (size=12151) 2024-11-13T19:23:27,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/e6353d4687824eaa91ca3f30332065e9 2024-11-13T19:23:27,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525867243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/607bf08ede824569bc6c39e4dcc87f7c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/607bf08ede824569bc6c39e4dcc87f7c 2024-11-13T19:23:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525867252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525867252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/607bf08ede824569bc6c39e4dcc87f7c, entries=250, sequenceid=215, filesize=16.5 K 2024-11-13T19:23:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5288daab5ef14f3f81471f1fca863c66 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5288daab5ef14f3f81471f1fca863c66 2024-11-13T19:23:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525867262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525867262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5288daab5ef14f3f81471f1fca863c66, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T19:23:27,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/e6353d4687824eaa91ca3f30332065e9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e6353d4687824eaa91ca3f30332065e9 2024-11-13T19:23:27,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e6353d4687824eaa91ca3f30332065e9, entries=150, sequenceid=215, filesize=11.9 K 2024-11-13T19:23:27,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 217ms, sequenceid=215, compaction requested=false 2024-11-13T19:23:27,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:27,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-13T19:23:27,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:27,310 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:23:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:23:27,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/0da35f1877f449f5b7f50da4a791ea09 is 50, key is test_row_0/A:col10/1731525807108/Put/seqid=0 2024-11-13T19:23:27,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741882_1058 (size=12151) 2024-11-13T19:23:27,352 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/0da35f1877f449f5b7f50da4a791ea09 2024-11-13T19:23:27,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/15792bce0287490baa5194a410dacdfe is 50, key is test_row_0/B:col10/1731525807108/Put/seqid=0 2024-11-13T19:23:27,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741883_1059 (size=12151) 2024-11-13T19:23:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:27,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:27,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525867474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525867476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525867478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525867480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525867483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525867593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525867593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525867593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525867593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:23:27,782 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/15792bce0287490baa5194a410dacdfe 2024-11-13T19:23:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525867791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/17e30f26bc3c4f5484416dc791c68639 is 50, key is test_row_0/C:col10/1731525807108/Put/seqid=0 2024-11-13T19:23:27,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741884_1060 (size=12151) 2024-11-13T19:23:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525867802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525867802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525867802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:27,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525867801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525868112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525868121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525868121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525868121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:23:28,204 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/17e30f26bc3c4f5484416dc791c68639 2024-11-13T19:23:28,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/0da35f1877f449f5b7f50da4a791ea09 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0da35f1877f449f5b7f50da4a791ea09 2024-11-13T19:23:28,219 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0da35f1877f449f5b7f50da4a791ea09, entries=150, sequenceid=239, filesize=11.9 K 2024-11-13T19:23:28,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/15792bce0287490baa5194a410dacdfe as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/15792bce0287490baa5194a410dacdfe 2024-11-13T19:23:28,227 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/15792bce0287490baa5194a410dacdfe, entries=150, sequenceid=239, filesize=11.9 K 2024-11-13T19:23:28,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/17e30f26bc3c4f5484416dc791c68639 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/17e30f26bc3c4f5484416dc791c68639 2024-11-13T19:23:28,237 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/17e30f26bc3c4f5484416dc791c68639, entries=150, sequenceid=239, filesize=11.9 K 2024-11-13T19:23:28,239 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 929ms, sequenceid=239, compaction requested=true 2024-11-13T19:23:28,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:28,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:28,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-13T19:23:28,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-13T19:23:28,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-13T19:23:28,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2460 sec 2024-11-13T19:23:28,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2540 sec 2024-11-13T19:23:28,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:28,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:28,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T19:23:28,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/f6d74fa6b0b0453c900ec5381d0c6195 is 50, key is test_row_0/A:col10/1731525808302/Put/seqid=0 2024-11-13T19:23:28,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741885_1061 (size=14541) 2024-11-13T19:23:28,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/f6d74fa6b0b0453c900ec5381d0c6195 2024-11-13T19:23:28,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e9c74a231ef54fc5953fc851d196c814 is 50, key is test_row_0/B:col10/1731525808302/Put/seqid=0 2024-11-13T19:23:28,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741886_1062 (size=12151) 2024-11-13T19:23:28,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e9c74a231ef54fc5953fc851d196c814 2024-11-13T19:23:28,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a79ceb6f2c744b58807fe746473c6e6d is 50, key is test_row_0/C:col10/1731525808302/Put/seqid=0 2024-11-13T19:23:28,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741887_1063 (size=12151) 2024-11-13T19:23:28,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a79ceb6f2c744b58807fe746473c6e6d 2024-11-13T19:23:28,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/f6d74fa6b0b0453c900ec5381d0c6195 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f6d74fa6b0b0453c900ec5381d0c6195 2024-11-13T19:23:28,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f6d74fa6b0b0453c900ec5381d0c6195, entries=200, sequenceid=252, filesize=14.2 K 2024-11-13T19:23:28,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e9c74a231ef54fc5953fc851d196c814 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e9c74a231ef54fc5953fc851d196c814 2024-11-13T19:23:28,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e9c74a231ef54fc5953fc851d196c814, entries=150, sequenceid=252, filesize=11.9 K 2024-11-13T19:23:28,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a79ceb6f2c744b58807fe746473c6e6d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a79ceb6f2c744b58807fe746473c6e6d 2024-11-13T19:23:28,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a79ceb6f2c744b58807fe746473c6e6d, entries=150, sequenceid=252, filesize=11.9 K 2024-11-13T19:23:28,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=120.76 KB/123660 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 114ms, sequenceid=252, compaction requested=true 2024-11-13T19:23:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:28,417 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:28,417 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:28,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:28,419 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:28,419 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56218 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:28,419 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:28,419 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:28,419 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:28,419 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:28,419 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/147b8e70b3d344898e74ff323c42d0f7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5288daab5ef14f3f81471f1fca863c66, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/15792bce0287490baa5194a410dacdfe, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e9c74a231ef54fc5953fc851d196c814] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=47.9 K 2024-11-13T19:23:28,419 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/ede83bb5010f41c3866e93415c08b893, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/607bf08ede824569bc6c39e4dcc87f7c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0da35f1877f449f5b7f50da4a791ea09, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f6d74fa6b0b0453c900ec5381d0c6195] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=54.9 K 2024-11-13T19:23:28,420 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 147b8e70b3d344898e74ff323c42d0f7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731525806097 2024-11-13T19:23:28,420 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede83bb5010f41c3866e93415c08b893, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731525806097 2024-11-13T19:23:28,420 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5288daab5ef14f3f81471f1fca863c66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731525806747 2024-11-13T19:23:28,420 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607bf08ede824569bc6c39e4dcc87f7c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731525806747 2024-11-13T19:23:28,420 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 15792bce0287490baa5194a410dacdfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731525807108 2024-11-13T19:23:28,421 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0da35f1877f449f5b7f50da4a791ea09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731525807108 2024-11-13T19:23:28,421 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c74a231ef54fc5953fc851d196c814, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525807465 2024-11-13T19:23:28,421 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6d74fa6b0b0453c900ec5381d0c6195, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525807465 2024-11-13T19:23:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:28,428 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:28,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T19:23:28,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/914b49f937a3475cae36b5522f3aa6b3 is 50, key is test_row_0/A:col10/1731525808422/Put/seqid=0 2024-11-13T19:23:28,438 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:28,438 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e282378b24de4466992a26696d694b21 is 50, key is test_row_0/B:col10/1731525808302/Put/seqid=0 2024-11-13T19:23:28,447 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:28,448 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/bbcb0707dbe242e5979b1c672aa17996 is 50, key is test_row_0/A:col10/1731525808302/Put/seqid=0 2024-11-13T19:23:28,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741890_1066 (size=12731) 2024-11-13T19:23:28,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741888_1064 (size=14741) 2024-11-13T19:23:28,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/914b49f937a3475cae36b5522f3aa6b3 2024-11-13T19:23:28,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741889_1065 (size=12731) 2024-11-13T19:23:28,473 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/bbcb0707dbe242e5979b1c672aa17996 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/bbcb0707dbe242e5979b1c672aa17996 2024-11-13T19:23:28,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/8f7972812fae4bdba83248cf882db643 is 50, key is test_row_0/B:col10/1731525808422/Put/seqid=0 2024-11-13T19:23:28,488 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e282378b24de4466992a26696d694b21 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e282378b24de4466992a26696d694b21 2024-11-13T19:23:28,490 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into bbcb0707dbe242e5979b1c672aa17996(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:28,490 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:28,491 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=12, startTime=1731525808417; duration=0sec 2024-11-13T19:23:28,491 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:28,491 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:28,491 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:28,497 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:28,497 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:28,497 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:28,497 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b5d6e2589086416893a7123681308e7b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e6353d4687824eaa91ca3f30332065e9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/17e30f26bc3c4f5484416dc791c68639, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a79ceb6f2c744b58807fe746473c6e6d] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=47.9 K 2024-11-13T19:23:28,498 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5d6e2589086416893a7123681308e7b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731525806097 2024-11-13T19:23:28,499 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6353d4687824eaa91ca3f30332065e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731525806747 2024-11-13T19:23:28,499 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into e282378b24de4466992a26696d694b21(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:28,500 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:28,500 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=12, startTime=1731525808417; duration=0sec 2024-11-13T19:23:28,500 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17e30f26bc3c4f5484416dc791c68639, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731525807108 2024-11-13T19:23:28,500 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:28,500 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:28,501 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79ceb6f2c744b58807fe746473c6e6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525807465 2024-11-13T19:23:28,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525868497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741891_1067 (size=12301) 2024-11-13T19:23:28,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/8f7972812fae4bdba83248cf882db643 2024-11-13T19:23:28,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5ca61fcc90b849f885f24d9ea343c022 is 50, key is test_row_0/C:col10/1731525808422/Put/seqid=0 2024-11-13T19:23:28,521 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:28,522 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a56d8fbc046c4223bdbc673329466476 is 50, key is test_row_0/C:col10/1731525808302/Put/seqid=0 2024-11-13T19:23:28,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741892_1068 (size=12301) 2024-11-13T19:23:28,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5ca61fcc90b849f885f24d9ea343c022 2024-11-13T19:23:28,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/914b49f937a3475cae36b5522f3aa6b3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/914b49f937a3475cae36b5522f3aa6b3 2024-11-13T19:23:28,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/914b49f937a3475cae36b5522f3aa6b3, entries=200, sequenceid=274, filesize=14.4 K 2024-11-13T19:23:28,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/8f7972812fae4bdba83248cf882db643 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/8f7972812fae4bdba83248cf882db643 2024-11-13T19:23:28,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741893_1069 (size=12731) 2024-11-13T19:23:28,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/8f7972812fae4bdba83248cf882db643, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T19:23:28,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5ca61fcc90b849f885f24d9ea343c022 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5ca61fcc90b849f885f24d9ea343c022 2024-11-13T19:23:28,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5ca61fcc90b849f885f24d9ea343c022, entries=150, sequenceid=274, filesize=12.0 K 2024-11-13T19:23:28,577 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 149ms, sequenceid=274, compaction requested=false 2024-11-13T19:23:28,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:28,617 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:28,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T19:23:28,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/59748f8195e14514b4a1a3a154208089 is 50, key is test_row_0/A:col10/1731525808470/Put/seqid=0 2024-11-13T19:23:28,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741894_1070 (size=14741) 2024-11-13T19:23:28,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525868661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525868668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525868677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525868679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525868687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525868783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525868782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525868792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525868792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525868802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,960 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a56d8fbc046c4223bdbc673329466476 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a56d8fbc046c4223bdbc673329466476 2024-11-13T19:23:28,969 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into a56d8fbc046c4223bdbc673329466476(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:28,973 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:28,973 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=12, startTime=1731525808417; duration=0sec 2024-11-13T19:23:28,974 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:28,974 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525868991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525868992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:28,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525869012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525869012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525869012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/59748f8195e14514b4a1a3a154208089 2024-11-13T19:23:29,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f492dfaf8ae74dc6bc257368a575784e is 50, key is test_row_0/B:col10/1731525808470/Put/seqid=0 2024-11-13T19:23:29,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741895_1071 (size=12301) 2024-11-13T19:23:29,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f492dfaf8ae74dc6bc257368a575784e 2024-11-13T19:23:29,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/b75456a96653436a8215934e57f95163 is 50, key is test_row_0/C:col10/1731525808470/Put/seqid=0 2024-11-13T19:23:29,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741896_1072 (size=12301) 2024-11-13T19:23:29,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/b75456a96653436a8215934e57f95163 2024-11-13T19:23:29,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/59748f8195e14514b4a1a3a154208089 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/59748f8195e14514b4a1a3a154208089 2024-11-13T19:23:29,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/59748f8195e14514b4a1a3a154208089, entries=200, sequenceid=291, filesize=14.4 K 2024-11-13T19:23:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-13T19:23:29,142 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:29,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f492dfaf8ae74dc6bc257368a575784e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f492dfaf8ae74dc6bc257368a575784e 2024-11-13T19:23:29,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-13T19:23:29,150 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:29,151 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:29,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:23:29,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f492dfaf8ae74dc6bc257368a575784e, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T19:23:29,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/b75456a96653436a8215934e57f95163 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b75456a96653436a8215934e57f95163 2024-11-13T19:23:29,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b75456a96653436a8215934e57f95163, entries=150, sequenceid=291, filesize=12.0 K 2024-11-13T19:23:29,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 564ms, sequenceid=291, compaction requested=true 2024-11-13T19:23:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:29,181 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:29,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:29,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:23:29,182 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:29,183 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:29,183 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:29,184 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:29,184 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:29,184 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,184 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,184 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e282378b24de4466992a26696d694b21, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/8f7972812fae4bdba83248cf882db643, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f492dfaf8ae74dc6bc257368a575784e] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.5 K 2024-11-13T19:23:29,184 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/bbcb0707dbe242e5979b1c672aa17996, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/914b49f937a3475cae36b5522f3aa6b3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/59748f8195e14514b4a1a3a154208089] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=41.2 K 2024-11-13T19:23:29,184 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbcb0707dbe242e5979b1c672aa17996, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525807465 2024-11-13T19:23:29,185 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e282378b24de4466992a26696d694b21, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525807465 2024-11-13T19:23:29,185 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914b49f937a3475cae36b5522f3aa6b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731525808392 2024-11-13T19:23:29,185 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f7972812fae4bdba83248cf882db643, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731525808410 2024-11-13T19:23:29,185 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59748f8195e14514b4a1a3a154208089, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731525808470 2024-11-13T19:23:29,186 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f492dfaf8ae74dc6bc257368a575784e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731525808470 2024-11-13T19:23:29,209 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:29,210 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c42e751f22824c888db480ba1fdeadae is 50, key is test_row_0/A:col10/1731525808470/Put/seqid=0 2024-11-13T19:23:29,213 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:29,214 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/0e36d305d58146d6908aa75901eb3f68 is 50, key is test_row_0/B:col10/1731525808470/Put/seqid=0 2024-11-13T19:23:29,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741897_1073 (size=12983) 2024-11-13T19:23:29,232 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c42e751f22824c888db480ba1fdeadae as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c42e751f22824c888db480ba1fdeadae 2024-11-13T19:23:29,255 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into c42e751f22824c888db480ba1fdeadae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:29,255 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:29,255 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525809181; duration=0sec 2024-11-13T19:23:29,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741898_1074 (size=12983) 2024-11-13T19:23:29,256 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:29,256 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:29,256 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:29,260 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:29,260 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:29,260 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,261 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a56d8fbc046c4223bdbc673329466476, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5ca61fcc90b849f885f24d9ea343c022, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b75456a96653436a8215934e57f95163] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.5 K 2024-11-13T19:23:29,261 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56d8fbc046c4223bdbc673329466476, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525807465 2024-11-13T19:23:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:23:29,262 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ca61fcc90b849f885f24d9ea343c022, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731525808410 2024-11-13T19:23:29,262 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75456a96653436a8215934e57f95163, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731525808470 2024-11-13T19:23:29,272 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:29,273 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5cbb4b75d6aa4b3987e656ede5a3e642 is 50, key is test_row_0/C:col10/1731525808470/Put/seqid=0 2024-11-13T19:23:29,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741899_1075 (size=12983) 2024-11-13T19:23:29,297 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5cbb4b75d6aa4b3987e656ede5a3e642 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5cbb4b75d6aa4b3987e656ede5a3e642 2024-11-13T19:23:29,306 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:29,306 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T19:23:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:29,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:29,310 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into 5cbb4b75d6aa4b3987e656ede5a3e642(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:29,310 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:29,310 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525809182; duration=0sec 2024-11-13T19:23:29,311 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:29,311 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/cef69c7e58784085b22b45df6ea19138 is 50, key is test_row_0/A:col10/1731525809304/Put/seqid=0 2024-11-13T19:23:29,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525869330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525869330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,336 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525869333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525869340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741900_1076 (size=14741) 2024-11-13T19:23:29,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525869342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/cef69c7e58784085b22b45df6ea19138 2024-11-13T19:23:29,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5ed76d359b034ccca41ceae08910fd0d is 50, key is test_row_0/B:col10/1731525809304/Put/seqid=0 2024-11-13T19:23:29,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741901_1077 (size=12301) 2024-11-13T19:23:29,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525869441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525869451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525869452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525869452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525869452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:23:29,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:29,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525869655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,663 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525869661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,666 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/0e36d305d58146d6908aa75901eb3f68 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/0e36d305d58146d6908aa75901eb3f68 2024-11-13T19:23:29,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525869662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525869662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525869662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,678 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into 0e36d305d58146d6908aa75901eb3f68(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:29,678 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:29,679 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525809181; duration=0sec 2024-11-13T19:23:29,679 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:29,679 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:29,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:23:29,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5ed76d359b034ccca41ceae08910fd0d 2024-11-13T19:23:29,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a5f89709de7e4210b4c5ec1b5525cd43 is 50, key is test_row_0/C:col10/1731525809304/Put/seqid=0 2024-11-13T19:23:29,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741902_1078 (size=12301) 2024-11-13T19:23:29,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:29,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:29,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:29,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525869971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525869972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525869972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525869975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:29,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:29,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525869981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:30,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:30,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:30,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:30,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:30,247 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a5f89709de7e4210b4c5ec1b5525cd43 2024-11-13T19:23:30,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:30,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:30,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:30,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:30,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/cef69c7e58784085b22b45df6ea19138 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/cef69c7e58784085b22b45df6ea19138 2024-11-13T19:23:30,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/cef69c7e58784085b22b45df6ea19138, entries=200, sequenceid=318, filesize=14.4 K 2024-11-13T19:23:30,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5ed76d359b034ccca41ceae08910fd0d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5ed76d359b034ccca41ceae08910fd0d 2024-11-13T19:23:30,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5ed76d359b034ccca41ceae08910fd0d, entries=150, sequenceid=318, filesize=12.0 K 2024-11-13T19:23:30,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a5f89709de7e4210b4c5ec1b5525cd43 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a5f89709de7e4210b4c5ec1b5525cd43 2024-11-13T19:23:30,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:23:30,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a5f89709de7e4210b4c5ec1b5525cd43, entries=150, sequenceid=318, filesize=12.0 K 2024-11-13T19:23:30,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 989ms, sequenceid=318, compaction requested=false 2024-11-13T19:23:30,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:30,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-13T19:23:30,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:30,408 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T19:23:30,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/2812e53da08b462d85169ad6d1c8c476 is 50, key is test_row_0/A:col10/1731525809330/Put/seqid=0 2024-11-13T19:23:30,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741903_1079 (size=12301) 2024-11-13T19:23:30,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:30,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:30,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525870517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525870519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525870519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525870520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525870521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525870632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525870632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525870632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525870633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525870641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,828 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/2812e53da08b462d85169ad6d1c8c476 2024-11-13T19:23:30,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525870843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,849 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/b5576ff45d564894abfbc99893065547 is 50, key is test_row_0/B:col10/1731525809330/Put/seqid=0 2024-11-13T19:23:30,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525870852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525870852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525870852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:30,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525870853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:30,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:30,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741904_1080 (size=12301) 2024-11-13T19:23:31,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525871162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525871172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525871181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525871202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525871202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:23:31,317 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/b5576ff45d564894abfbc99893065547 2024-11-13T19:23:31,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14 is 50, key is test_row_0/C:col10/1731525809330/Put/seqid=0 2024-11-13T19:23:31,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741905_1081 (size=12301) 2024-11-13T19:23:31,339 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14 2024-11-13T19:23:31,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/2812e53da08b462d85169ad6d1c8c476 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/2812e53da08b462d85169ad6d1c8c476 2024-11-13T19:23:31,359 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/2812e53da08b462d85169ad6d1c8c476, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T19:23:31,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/b5576ff45d564894abfbc99893065547 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b5576ff45d564894abfbc99893065547 2024-11-13T19:23:31,376 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b5576ff45d564894abfbc99893065547, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T19:23:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14 2024-11-13T19:23:31,390 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14, entries=150, sequenceid=332, filesize=12.0 K 2024-11-13T19:23:31,403 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 996ms, sequenceid=332, compaction requested=true 2024-11-13T19:23:31,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:31,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:31,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-13T19:23:31,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-13T19:23:31,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-13T19:23:31,412 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2570 sec 2024-11-13T19:23:31,415 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.2660 sec 2024-11-13T19:23:31,674 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:31,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T19:23:31,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:31,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/0a5da6f687c6400387dfa66d3cfc73d9 is 50, key is test_row_0/A:col10/1731525810518/Put/seqid=0 2024-11-13T19:23:31,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525871700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525871705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,708 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525871707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525871712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525871712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741906_1082 (size=14741) 2024-11-13T19:23:31,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/0a5da6f687c6400387dfa66d3cfc73d9 2024-11-13T19:23:31,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/789ddb0f8fbb46b8906a680cc13bd8b2 is 50, key is test_row_0/B:col10/1731525810518/Put/seqid=0 2024-11-13T19:23:31,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741907_1083 (size=12301) 2024-11-13T19:23:31,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/789ddb0f8fbb46b8906a680cc13bd8b2 2024-11-13T19:23:31,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/348daf40132c47ffb2a57ee22cb45e00 is 50, key is test_row_0/C:col10/1731525810518/Put/seqid=0 2024-11-13T19:23:31,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741908_1084 (size=12301) 2024-11-13T19:23:31,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/348daf40132c47ffb2a57ee22cb45e00 2024-11-13T19:23:31,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/0a5da6f687c6400387dfa66d3cfc73d9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0a5da6f687c6400387dfa66d3cfc73d9 2024-11-13T19:23:31,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0a5da6f687c6400387dfa66d3cfc73d9, entries=200, sequenceid=357, filesize=14.4 K 2024-11-13T19:23:31,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/789ddb0f8fbb46b8906a680cc13bd8b2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/789ddb0f8fbb46b8906a680cc13bd8b2 2024-11-13T19:23:31,802 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/789ddb0f8fbb46b8906a680cc13bd8b2, entries=150, sequenceid=357, filesize=12.0 K 2024-11-13T19:23:31,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/348daf40132c47ffb2a57ee22cb45e00 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/348daf40132c47ffb2a57ee22cb45e00 2024-11-13T19:23:31,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/348daf40132c47ffb2a57ee22cb45e00, entries=150, sequenceid=357, filesize=12.0 K 2024-11-13T19:23:31,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 138ms, sequenceid=357, compaction requested=true 2024-11-13T19:23:31,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:31,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:31,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:31,814 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:31,814 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:31,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:31,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:31,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:31,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:31,816 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54766 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:31,816 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:31,816 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:31,816 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:31,816 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:31,816 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/0e36d305d58146d6908aa75901eb3f68, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5ed76d359b034ccca41ceae08910fd0d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b5576ff45d564894abfbc99893065547, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/789ddb0f8fbb46b8906a680cc13bd8b2] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=48.7 K 2024-11-13T19:23:31,816 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:31,817 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c42e751f22824c888db480ba1fdeadae, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/cef69c7e58784085b22b45df6ea19138, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/2812e53da08b462d85169ad6d1c8c476, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0a5da6f687c6400387dfa66d3cfc73d9] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=53.5 K 2024-11-13T19:23:31,817 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e36d305d58146d6908aa75901eb3f68, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731525808470 2024-11-13T19:23:31,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:31,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:31,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T19:23:31,821 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c42e751f22824c888db480ba1fdeadae, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731525808470 2024-11-13T19:23:31,822 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed76d359b034ccca41ceae08910fd0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731525808671 2024-11-13T19:23:31,822 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef69c7e58784085b22b45df6ea19138, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731525808654 2024-11-13T19:23:31,823 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting b5576ff45d564894abfbc99893065547, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731525809326 2024-11-13T19:23:31,823 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2812e53da08b462d85169ad6d1c8c476, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731525809326 2024-11-13T19:23:31,824 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 789ddb0f8fbb46b8906a680cc13bd8b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731525810512 2024-11-13T19:23:31,824 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a5da6f687c6400387dfa66d3cfc73d9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731525810512 2024-11-13T19:23:31,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c6d77f39ed60404aab44696d98aeeeca is 50, key is test_row_0/A:col10/1731525811702/Put/seqid=0 2024-11-13T19:23:31,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741909_1085 (size=12301) 2024-11-13T19:23:31,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c6d77f39ed60404aab44696d98aeeeca 2024-11-13T19:23:31,861 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:31,862 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5c1024bfbb944a93b33b2757c4fd6465 is 50, key is test_row_0/A:col10/1731525810518/Put/seqid=0 2024-11-13T19:23:31,865 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:31,866 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/512a02cc63cf480aab5520dc53bff82d is 50, key is test_row_0/B:col10/1731525810518/Put/seqid=0 2024-11-13T19:23:31,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/924101d559c8400ebd195e453c3aba83 is 50, key is test_row_0/B:col10/1731525811702/Put/seqid=0 2024-11-13T19:23:31,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741910_1086 (size=13119) 2024-11-13T19:23:31,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741912_1088 (size=12301) 2024-11-13T19:23:31,893 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5c1024bfbb944a93b33b2757c4fd6465 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5c1024bfbb944a93b33b2757c4fd6465 2024-11-13T19:23:31,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/924101d559c8400ebd195e453c3aba83 2024-11-13T19:23:31,904 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into 5c1024bfbb944a93b33b2757c4fd6465(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:31,904 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:31,904 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=12, startTime=1731525811813; duration=0sec 2024-11-13T19:23:31,904 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:31,904 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:31,905 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:23:31,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525871897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525871899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,908 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:23:31,908 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:31,908 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:31,908 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5cbb4b75d6aa4b3987e656ede5a3e642, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a5f89709de7e4210b4c5ec1b5525cd43, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/348daf40132c47ffb2a57ee22cb45e00] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=48.7 K 2024-11-13T19:23:31,909 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cbb4b75d6aa4b3987e656ede5a3e642, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731525808470 2024-11-13T19:23:31,909 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5f89709de7e4210b4c5ec1b5525cd43, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731525808671 2024-11-13T19:23:31,909 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9ed0ecea1ef475c8ec46f9ad5c6bf14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731525809326 2024-11-13T19:23:31,910 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 348daf40132c47ffb2a57ee22cb45e00, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731525810512 2024-11-13T19:23:31,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741911_1087 (size=13119) 2024-11-13T19:23:31,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:31,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525871906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:31,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:31,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/9e831f1f0edf42bb8dcadc0a5aacff95 is 50, key is test_row_0/C:col10/1731525811702/Put/seqid=0 2024-11-13T19:23:31,926 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/512a02cc63cf480aab5520dc53bff82d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/512a02cc63cf480aab5520dc53bff82d 2024-11-13T19:23:31,946 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into 512a02cc63cf480aab5520dc53bff82d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:31,946 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:31,946 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=12, startTime=1731525811814; duration=0sec 2024-11-13T19:23:31,947 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:31,947 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:31,967 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#77 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:31,971 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/d0a4585583604cd0911869ce64c3295b is 50, key is test_row_0/C:col10/1731525810518/Put/seqid=0 2024-11-13T19:23:31,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741913_1089 (size=12301) 2024-11-13T19:23:32,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741914_1090 (size=13119) 2024-11-13T19:23:32,009 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/d0a4585583604cd0911869ce64c3295b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/d0a4585583604cd0911869ce64c3295b 2024-11-13T19:23:32,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525872012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525872012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,020 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into d0a4585583604cd0911869ce64c3295b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:32,020 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:32,020 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=12, startTime=1731525811814; duration=0sec 2024-11-13T19:23:32,020 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:32,020 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525872022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525872231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525872231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525872231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/9e831f1f0edf42bb8dcadc0a5aacff95 2024-11-13T19:23:32,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c6d77f39ed60404aab44696d98aeeeca as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c6d77f39ed60404aab44696d98aeeeca 2024-11-13T19:23:32,391 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c6d77f39ed60404aab44696d98aeeeca, entries=150, sequenceid=370, filesize=12.0 K 2024-11-13T19:23:32,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/924101d559c8400ebd195e453c3aba83 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/924101d559c8400ebd195e453c3aba83 2024-11-13T19:23:32,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/924101d559c8400ebd195e453c3aba83, entries=150, sequenceid=370, filesize=12.0 K 2024-11-13T19:23:32,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/9e831f1f0edf42bb8dcadc0a5aacff95 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e831f1f0edf42bb8dcadc0a5aacff95 2024-11-13T19:23:32,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e831f1f0edf42bb8dcadc0a5aacff95, entries=150, sequenceid=370, filesize=12.0 K 2024-11-13T19:23:32,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 590ms, sequenceid=370, compaction requested=false 2024-11-13T19:23:32,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:32,543 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:32,543 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T19:23:32,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5cdb867ae8904d539ec7ef6fb69d9a78 is 50, key is test_row_0/A:col10/1731525811903/Put/seqid=0 2024-11-13T19:23:32,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525872561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525872561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,564 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741915_1091 (size=12301) 2024-11-13T19:23:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525872563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525872678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525872678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525872678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,685 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525872732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525872732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,744 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 1334c8616b794fd8c4fb42c4bfbc0d50 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:32,742 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 1334c8616b794fd8c4fb42c4bfbc0d50 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525872892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525872892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525872892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:32,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:32,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5cdb867ae8904d539ec7ef6fb69d9a78 2024-11-13T19:23:32,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5a312603fd5c4ceba6ebb6dde999896f is 50, key is test_row_0/B:col10/1731525811903/Put/seqid=0 2024-11-13T19:23:32,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741916_1092 (size=12301) 2024-11-13T19:23:33,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525873202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525873202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525873202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,205 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-13T19:23:33,322 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:33,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-13T19:23:33,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T19:23:33,324 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:33,325 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:33,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:33,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5a312603fd5c4ceba6ebb6dde999896f 2024-11-13T19:23:33,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/e907d55608994c988f574862ef0e7d11 is 50, key is test_row_0/C:col10/1731525811903/Put/seqid=0 2024-11-13T19:23:33,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741917_1093 (size=12301) 2024-11-13T19:23:33,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=398 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/e907d55608994c988f574862ef0e7d11 2024-11-13T19:23:33,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5cdb867ae8904d539ec7ef6fb69d9a78 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5cdb867ae8904d539ec7ef6fb69d9a78 2024-11-13T19:23:33,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5cdb867ae8904d539ec7ef6fb69d9a78, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T19:23:33,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/5a312603fd5c4ceba6ebb6dde999896f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5a312603fd5c4ceba6ebb6dde999896f 2024-11-13T19:23:33,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5a312603fd5c4ceba6ebb6dde999896f, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T19:23:33,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/e907d55608994c988f574862ef0e7d11 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e907d55608994c988f574862ef0e7d11 2024-11-13T19:23:33,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e907d55608994c988f574862ef0e7d11, entries=150, sequenceid=398, filesize=12.0 K 2024-11-13T19:23:33,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 881ms, sequenceid=398, compaction requested=true 2024-11-13T19:23:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:33,424 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:33,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:33,425 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:33,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:33,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:33,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:33,426 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:33,426 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:33,426 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:33,426 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:33,426 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:33,426 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:33,426 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5c1024bfbb944a93b33b2757c4fd6465, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c6d77f39ed60404aab44696d98aeeeca, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5cdb867ae8904d539ec7ef6fb69d9a78] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.8 K 2024-11-13T19:23:33,426 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/512a02cc63cf480aab5520dc53bff82d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/924101d559c8400ebd195e453c3aba83, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5a312603fd5c4ceba6ebb6dde999896f] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.8 K 2024-11-13T19:23:33,427 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1024bfbb944a93b33b2757c4fd6465, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731525810512 2024-11-13T19:23:33,427 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 512a02cc63cf480aab5520dc53bff82d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731525810512 2024-11-13T19:23:33,427 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6d77f39ed60404aab44696d98aeeeca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731525811702 2024-11-13T19:23:33,427 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 924101d559c8400ebd195e453c3aba83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731525811702 2024-11-13T19:23:33,428 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a312603fd5c4ceba6ebb6dde999896f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731525811894 2024-11-13T19:23:33,428 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cdb867ae8904d539ec7ef6fb69d9a78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731525811894 2024-11-13T19:23:33,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T19:23:33,437 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:33,438 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/40a970596c354051990958d0b012f0da is 50, key is test_row_0/B:col10/1731525811903/Put/seqid=0 2024-11-13T19:23:33,442 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:33,443 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/523afef9f5674cfa8e159dfbbcffbd54 is 50, key is test_row_0/A:col10/1731525811903/Put/seqid=0 2024-11-13T19:23:33,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741918_1094 (size=13221) 2024-11-13T19:23:33,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741919_1095 (size=13221) 2024-11-13T19:23:33,465 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/523afef9f5674cfa8e159dfbbcffbd54 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/523afef9f5674cfa8e159dfbbcffbd54 2024-11-13T19:23:33,471 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into 523afef9f5674cfa8e159dfbbcffbd54(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:33,471 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:33,472 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525813424; duration=0sec 2024-11-13T19:23:33,472 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:33,472 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:33,472 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:33,473 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:33,473 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:33,474 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:33,474 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/d0a4585583604cd0911869ce64c3295b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e831f1f0edf42bb8dcadc0a5aacff95, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e907d55608994c988f574862ef0e7d11] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.8 K 2024-11-13T19:23:33,475 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0a4585583604cd0911869ce64c3295b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1731525810512 2024-11-13T19:23:33,475 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e831f1f0edf42bb8dcadc0a5aacff95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1731525811702 2024-11-13T19:23:33,476 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e907d55608994c988f574862ef0e7d11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731525811894 2024-11-13T19:23:33,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-13T19:23:33,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:33,478 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:23:33,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c1bad009ace14e32a681fc1bac6f36d0 is 50, key is test_row_0/A:col10/1731525812559/Put/seqid=0 2024-11-13T19:23:33,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741920_1096 (size=12301) 2024-11-13T19:23:33,494 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c1bad009ace14e32a681fc1bac6f36d0 2024-11-13T19:23:33,494 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:33,495 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a3eb898747bf4f249e29bb84afd990a4 is 50, key is test_row_0/C:col10/1731525811903/Put/seqid=0 2024-11-13T19:23:33,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/509f02562e514ebe9bad8f442d856bfa is 50, key is test_row_0/B:col10/1731525812559/Put/seqid=0 2024-11-13T19:23:33,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741921_1097 (size=13221) 2024-11-13T19:23:33,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741922_1098 (size=12301) 2024-11-13T19:23:33,527 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/509f02562e514ebe9bad8f442d856bfa 2024-11-13T19:23:33,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/09cf06f25fd24b60ae4f4b9a5b31f9c7 is 50, key is test_row_0/C:col10/1731525812559/Put/seqid=0 2024-11-13T19:23:33,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741923_1099 (size=12301) 2024-11-13T19:23:33,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T19:23:33,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:33,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:33,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525873770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525873772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525873772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,857 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/40a970596c354051990958d0b012f0da as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/40a970596c354051990958d0b012f0da 2024-11-13T19:23:33,862 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into 40a970596c354051990958d0b012f0da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:33,862 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:33,863 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525813424; duration=0sec 2024-11-13T19:23:33,863 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:33,863 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:33,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525873881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525873891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525873892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,895 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:33,921 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/a3eb898747bf4f249e29bb84afd990a4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a3eb898747bf4f249e29bb84afd990a4 2024-11-13T19:23:33,930 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into a3eb898747bf4f249e29bb84afd990a4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:33,930 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:33,930 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525813425; duration=0sec 2024-11-13T19:23:33,930 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:33,930 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:33,949 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/09cf06f25fd24b60ae4f4b9a5b31f9c7 2024-11-13T19:23:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T19:23:33,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/c1bad009ace14e32a681fc1bac6f36d0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c1bad009ace14e32a681fc1bac6f36d0 2024-11-13T19:23:33,962 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c1bad009ace14e32a681fc1bac6f36d0, entries=150, sequenceid=410, filesize=12.0 K 2024-11-13T19:23:33,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/509f02562e514ebe9bad8f442d856bfa as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/509f02562e514ebe9bad8f442d856bfa 2024-11-13T19:23:33,969 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/509f02562e514ebe9bad8f442d856bfa, entries=150, sequenceid=410, filesize=12.0 K 2024-11-13T19:23:33,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/09cf06f25fd24b60ae4f4b9a5b31f9c7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/09cf06f25fd24b60ae4f4b9a5b31f9c7 2024-11-13T19:23:33,976 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/09cf06f25fd24b60ae4f4b9a5b31f9c7, entries=150, sequenceid=410, filesize=12.0 K 2024-11-13T19:23:33,978 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 500ms, sequenceid=410, compaction requested=false 2024-11-13T19:23:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:33,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-13T19:23:33,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-13T19:23:33,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-13T19:23:33,982 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 655 msec 2024-11-13T19:23:33,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 660 msec 2024-11-13T19:23:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:34,095 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:34,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T19:23:34,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b4493639941140c19e06b4e84a0cff63 is 50, key is test_row_0/A:col10/1731525814094/Put/seqid=0 2024-11-13T19:23:34,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741924_1100 (size=12301) 2024-11-13T19:23:34,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525874106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525874108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525874108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525874221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,225 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525874221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525874226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525874431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525874432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525874432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-13T19:23:34,462 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:34,462 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:34,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-13T19:23:34,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T19:23:34,464 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:34,465 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:34,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:34,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b4493639941140c19e06b4e84a0cff63 2024-11-13T19:23:34,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/660f86d396c04de3a4d4e5963b03ef6a is 50, key is test_row_0/B:col10/1731525814094/Put/seqid=0 2024-11-13T19:23:34,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741925_1101 (size=12301) 2024-11-13T19:23:34,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/660f86d396c04de3a4d4e5963b03ef6a 2024-11-13T19:23:34,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6 is 50, key is test_row_0/C:col10/1731525814094/Put/seqid=0 2024-11-13T19:23:34,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741926_1102 (size=12301) 2024-11-13T19:23:34,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T19:23:34,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T19:23:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525874742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525874741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525874742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,746 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525874761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,765 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 1334c8616b794fd8c4fb42c4bfbc0d50 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T19:23:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:34,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525874771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:34,772 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 1334c8616b794fd8c4fb42c4bfbc0d50 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:34,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T19:23:34,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T19:23:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:34,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6 2024-11-13T19:23:34,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b4493639941140c19e06b4e84a0cff63 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b4493639941140c19e06b4e84a0cff63 2024-11-13T19:23:34,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b4493639941140c19e06b4e84a0cff63, entries=150, sequenceid=439, filesize=12.0 K 2024-11-13T19:23:34,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/660f86d396c04de3a4d4e5963b03ef6a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/660f86d396c04de3a4d4e5963b03ef6a 2024-11-13T19:23:34,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/660f86d396c04de3a4d4e5963b03ef6a, entries=150, sequenceid=439, filesize=12.0 K 2024-11-13T19:23:34,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6 2024-11-13T19:23:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6, entries=150, sequenceid=439, filesize=12.0 K 2024-11-13T19:23:34,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 880ms, sequenceid=439, compaction requested=true 2024-11-13T19:23:34,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:34,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:34,975 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:34,975 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:34,976 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:34,977 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:34,977 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:34,977 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:34,977 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:34,977 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,977 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:34,977 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/523afef9f5674cfa8e159dfbbcffbd54, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c1bad009ace14e32a681fc1bac6f36d0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b4493639941140c19e06b4e84a0cff63] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.9 K 2024-11-13T19:23:34,977 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/40a970596c354051990958d0b012f0da, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/509f02562e514ebe9bad8f442d856bfa, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/660f86d396c04de3a4d4e5963b03ef6a] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.9 K 2024-11-13T19:23:34,978 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 40a970596c354051990958d0b012f0da, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731525811894 2024-11-13T19:23:34,978 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 523afef9f5674cfa8e159dfbbcffbd54, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731525811894 2024-11-13T19:23:34,983 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1bad009ace14e32a681fc1bac6f36d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731525812557 2024-11-13T19:23:34,983 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 509f02562e514ebe9bad8f442d856bfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731525812557 2024-11-13T19:23:34,985 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4493639941140c19e06b4e84a0cff63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731525813765 2024-11-13T19:23:34,985 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 660f86d396c04de3a4d4e5963b03ef6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731525813765 2024-11-13T19:23:35,015 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#90 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:35,016 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/06a1cebb7a454ab8af1aea3777a6811d is 50, key is test_row_0/B:col10/1731525814094/Put/seqid=0 2024-11-13T19:23:35,021 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#91 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:35,021 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/13b0b0ef816242c597b232ee6f1ab559 is 50, key is test_row_0/A:col10/1731525814094/Put/seqid=0 2024-11-13T19:23:35,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741927_1103 (size=13323) 2024-11-13T19:23:35,056 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/06a1cebb7a454ab8af1aea3777a6811d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/06a1cebb7a454ab8af1aea3777a6811d 2024-11-13T19:23:35,065 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into 06a1cebb7a454ab8af1aea3777a6811d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:35,065 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:35,066 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525814975; duration=0sec 2024-11-13T19:23:35,066 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:35,066 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:35,066 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:35,067 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:35,067 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:35,067 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,068 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a3eb898747bf4f249e29bb84afd990a4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/09cf06f25fd24b60ae4f4b9a5b31f9c7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=36.9 K 2024-11-13T19:23:35,069 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a3eb898747bf4f249e29bb84afd990a4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=398, earliestPutTs=1731525811894 2024-11-13T19:23:35,070 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 09cf06f25fd24b60ae4f4b9a5b31f9c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=410, earliestPutTs=1731525812557 2024-11-13T19:23:35,071 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e097ffbad8a4e8a8d5b6d9f75c77ff6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731525813765 2024-11-13T19:23:35,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-13T19:23:35,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,080 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-13T19:23:35,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741928_1104 (size=13323) 2024-11-13T19:23:35,091 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/13b0b0ef816242c597b232ee6f1ab559 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/13b0b0ef816242c597b232ee6f1ab559 2024-11-13T19:23:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T19:23:35,092 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#92 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:35,093 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/bd8432bba5d4450ab525035ac4d2ba8a is 50, key is test_row_0/C:col10/1731525814094/Put/seqid=0 2024-11-13T19:23:35,100 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into 13b0b0ef816242c597b232ee6f1ab559(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:35,100 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:35,100 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525814975; duration=0sec 2024-11-13T19:23:35,100 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:35,100 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:35,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/3e8fbdf911cf4a529ae2acb80038271c is 50, key is test_row_0/A:col10/1731525814106/Put/seqid=0 2024-11-13T19:23:35,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741929_1105 (size=13323) 2024-11-13T19:23:35,141 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/bd8432bba5d4450ab525035ac4d2ba8a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bd8432bba5d4450ab525035ac4d2ba8a 2024-11-13T19:23:35,149 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into bd8432bba5d4450ab525035ac4d2ba8a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:35,149 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:35,149 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525814976; duration=0sec 2024-11-13T19:23:35,149 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:35,149 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:35,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741930_1106 (size=9857) 2024-11-13T19:23:35,158 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/3e8fbdf911cf4a529ae2acb80038271c 2024-11-13T19:23:35,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/90e5a8a3dc72478bad0a6daed96cf1b5 is 50, key is test_row_0/B:col10/1731525814106/Put/seqid=0 2024-11-13T19:23:35,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741931_1107 (size=9857) 2024-11-13T19:23:35,210 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/90e5a8a3dc72478bad0a6daed96cf1b5 2024-11-13T19:23:35,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/c210d2d6821b42a0b3e1d2ba04220c16 is 50, key is test_row_0/C:col10/1731525814106/Put/seqid=0 2024-11-13T19:23:35,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741932_1108 (size=9857) 2024-11-13T19:23:35,256 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/c210d2d6821b42a0b3e1d2ba04220c16 2024-11-13T19:23:35,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:35,259 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:35,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/3e8fbdf911cf4a529ae2acb80038271c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/3e8fbdf911cf4a529ae2acb80038271c 2024-11-13T19:23:35,280 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/3e8fbdf911cf4a529ae2acb80038271c, entries=100, sequenceid=449, filesize=9.6 K 2024-11-13T19:23:35,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/90e5a8a3dc72478bad0a6daed96cf1b5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/90e5a8a3dc72478bad0a6daed96cf1b5 2024-11-13T19:23:35,294 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/90e5a8a3dc72478bad0a6daed96cf1b5, entries=100, sequenceid=449, filesize=9.6 K 2024-11-13T19:23:35,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/c210d2d6821b42a0b3e1d2ba04220c16 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/c210d2d6821b42a0b3e1d2ba04220c16 2024-11-13T19:23:35,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/c210d2d6821b42a0b3e1d2ba04220c16, entries=100, sequenceid=449, filesize=9.6 K 2024-11-13T19:23:35,307 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=147.60 KB/151140 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 227ms, sequenceid=449, compaction requested=false 2024-11-13T19:23:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-13T19:23:35,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:35,307 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:35,308 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T19:23:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-13T19:23:35,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-13T19:23:35,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 845 msec 2024-11-13T19:23:35,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 855 msec 2024-11-13T19:23:35,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5bf89365abec4d058c935f9b5cbc1619 is 50, key is test_row_0/A:col10/1731525815302/Put/seqid=0 2024-11-13T19:23:35,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525875325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525875325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525875326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741933_1109 (size=17181) 2024-11-13T19:23:35,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5bf89365abec4d058c935f9b5cbc1619 2024-11-13T19:23:35,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e7769e6f46b34142a2d1c6fa4cde539d is 50, key is test_row_0/B:col10/1731525815302/Put/seqid=0 2024-11-13T19:23:35,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741934_1110 (size=12301) 2024-11-13T19:23:35,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525875442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525875442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525875442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-13T19:23:35,602 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:35,602 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-13T19:23:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T19:23:35,605 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:35,605 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:35,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:35,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525875651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525875651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525875651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:35,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:35,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T19:23:35,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T19:23:35,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:35,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:35,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:35,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:35,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e7769e6f46b34142a2d1c6fa4cde539d 2024-11-13T19:23:35,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/2dd74e80d193434c85179aba88a871d4 is 50, key is test_row_0/C:col10/1731525815302/Put/seqid=0 2024-11-13T19:23:35,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741935_1111 (size=12301) 2024-11-13T19:23:35,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/2dd74e80d193434c85179aba88a871d4 2024-11-13T19:23:35,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/5bf89365abec4d058c935f9b5cbc1619 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5bf89365abec4d058c935f9b5cbc1619 2024-11-13T19:23:35,876 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5bf89365abec4d058c935f9b5cbc1619, entries=250, sequenceid=477, filesize=16.8 K 2024-11-13T19:23:35,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/e7769e6f46b34142a2d1c6fa4cde539d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e7769e6f46b34142a2d1c6fa4cde539d 2024-11-13T19:23:35,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e7769e6f46b34142a2d1c6fa4cde539d, entries=150, sequenceid=477, filesize=12.0 K 2024-11-13T19:23:35,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/2dd74e80d193434c85179aba88a871d4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/2dd74e80d193434c85179aba88a871d4 2024-11-13T19:23:35,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/2dd74e80d193434c85179aba88a871d4, entries=150, sequenceid=477, filesize=12.0 K 2024-11-13T19:23:35,895 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 587ms, sequenceid=477, compaction requested=true 2024-11-13T19:23:35,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:35,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:35,895 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:35,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:35,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:35,895 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:35,895 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:35,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:35,896 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:35,897 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:35,897 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,897 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/06a1cebb7a454ab8af1aea3777a6811d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/90e5a8a3dc72478bad0a6daed96cf1b5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e7769e6f46b34142a2d1c6fa4cde539d] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=34.6 K 2024-11-13T19:23:35,897 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40361 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:35,897 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:35,897 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,897 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/13b0b0ef816242c597b232ee6f1ab559, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/3e8fbdf911cf4a529ae2acb80038271c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5bf89365abec4d058c935f9b5cbc1619] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=39.4 K 2024-11-13T19:23:35,898 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13b0b0ef816242c597b232ee6f1ab559, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731525813765 2024-11-13T19:23:35,898 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e8fbdf911cf4a529ae2acb80038271c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731525814104 2024-11-13T19:23:35,898 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 06a1cebb7a454ab8af1aea3777a6811d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731525813765 2024-11-13T19:23:35,899 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 90e5a8a3dc72478bad0a6daed96cf1b5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731525814104 2024-11-13T19:23:35,899 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf89365abec4d058c935f9b5cbc1619, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731525815285 2024-11-13T19:23:35,899 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e7769e6f46b34142a2d1c6fa4cde539d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731525815302 2024-11-13T19:23:35,911 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#99 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:35,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-13T19:23:35,912 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/70fbf41bb4e849159e9515d24437dbde is 50, key is test_row_0/A:col10/1731525815302/Put/seqid=0 2024-11-13T19:23:35,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:35,912 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:23:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T19:23:35,925 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#100 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:35,926 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/287f2f2734b847009ac7c778c14f726d is 50, key is test_row_0/B:col10/1731525815302/Put/seqid=0 2024-11-13T19:23:35,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/67edc54150b5432396ca37da60ef410b is 50, key is test_row_0/A:col10/1731525815323/Put/seqid=0 2024-11-13T19:23:35,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:35,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:35,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741937_1113 (size=13425) 2024-11-13T19:23:35,990 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/287f2f2734b847009ac7c778c14f726d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/287f2f2734b847009ac7c778c14f726d 2024-11-13T19:23:35,999 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into 287f2f2734b847009ac7c778c14f726d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:35,999 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:35,999 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525815895; duration=0sec 2024-11-13T19:23:35,999 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:35,999 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:35,999 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:36,005 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:36,006 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:36,006 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:36,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741936_1112 (size=13425) 2024-11-13T19:23:36,006 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bd8432bba5d4450ab525035ac4d2ba8a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/c210d2d6821b42a0b3e1d2ba04220c16, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/2dd74e80d193434c85179aba88a871d4] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=34.6 K 2024-11-13T19:23:36,008 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting bd8432bba5d4450ab525035ac4d2ba8a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731525813765 2024-11-13T19:23:36,009 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c210d2d6821b42a0b3e1d2ba04220c16, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1731525814104 2024-11-13T19:23:36,009 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd74e80d193434c85179aba88a871d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731525815302 2024-11-13T19:23:36,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741938_1114 (size=12301) 2024-11-13T19:23:36,046 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:36,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525876044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525876044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,050 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/69d05a97d6214594bf378893f373d010 is 50, key is test_row_0/C:col10/1731525815302/Put/seqid=0 2024-11-13T19:23:36,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525876049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,055 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741939_1115 (size=13425) 2024-11-13T19:23:36,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525876152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525876163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525876163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T19:23:36,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525876362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525876371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525876371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,375 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,412 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/70fbf41bb4e849159e9515d24437dbde as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/70fbf41bb4e849159e9515d24437dbde 2024-11-13T19:23:36,420 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into 70fbf41bb4e849159e9515d24437dbde(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:36,420 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:36,421 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525815895; duration=0sec 2024-11-13T19:23:36,421 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:36,421 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:36,428 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/67edc54150b5432396ca37da60ef410b 2024-11-13T19:23:36,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/2c9b5ee7120149e5881e0b7b270455b6 is 50, key is test_row_0/B:col10/1731525815323/Put/seqid=0 2024-11-13T19:23:36,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741940_1116 (size=12301) 2024-11-13T19:23:36,456 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/2c9b5ee7120149e5881e0b7b270455b6 2024-11-13T19:23:36,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/6ab3cd4bd40a40c3bd38196a7e044547 is 50, key is test_row_0/C:col10/1731525815323/Put/seqid=0 2024-11-13T19:23:36,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741941_1117 (size=12301) 2024-11-13T19:23:36,485 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/69d05a97d6214594bf378893f373d010 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/69d05a97d6214594bf378893f373d010 2024-11-13T19:23:36,492 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into 69d05a97d6214594bf378893f373d010(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:36,492 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:36,492 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525815895; duration=0sec 2024-11-13T19:23:36,492 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:36,492 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:36,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525876671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525876681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525876681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:36,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:36,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T19:23:36,871 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=488 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/6ab3cd4bd40a40c3bd38196a7e044547 2024-11-13T19:23:36,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/67edc54150b5432396ca37da60ef410b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/67edc54150b5432396ca37da60ef410b 2024-11-13T19:23:36,883 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/67edc54150b5432396ca37da60ef410b, entries=150, sequenceid=488, filesize=12.0 K 2024-11-13T19:23:36,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/2c9b5ee7120149e5881e0b7b270455b6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2c9b5ee7120149e5881e0b7b270455b6 2024-11-13T19:23:36,890 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2c9b5ee7120149e5881e0b7b270455b6, entries=150, sequenceid=488, filesize=12.0 K 2024-11-13T19:23:36,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/6ab3cd4bd40a40c3bd38196a7e044547 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6ab3cd4bd40a40c3bd38196a7e044547 2024-11-13T19:23:36,902 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6ab3cd4bd40a40c3bd38196a7e044547, entries=150, sequenceid=488, filesize=12.0 K 2024-11-13T19:23:36,903 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 991ms, sequenceid=488, compaction requested=false 2024-11-13T19:23:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:36,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-13T19:23:36,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-13T19:23:36,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-13T19:23:36,909 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-11-13T19:23:36,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.3070 sec 2024-11-13T19:23:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:37,196 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:37,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T19:23:37,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525877208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525877209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525877209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/72cf3a4705a748cf92f8a7ff9792a2e0 is 50, key is test_row_0/A:col10/1731525817194/Put/seqid=0 2024-11-13T19:23:37,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741942_1118 (size=12301) 2024-11-13T19:23:37,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525877322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525877322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525877322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525877532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525877532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,537 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525877532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/72cf3a4705a748cf92f8a7ff9792a2e0 2024-11-13T19:23:37,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/2ebdd3d3e81649038bcb3ddd5a54fab2 is 50, key is test_row_0/B:col10/1731525817194/Put/seqid=0 2024-11-13T19:23:37,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741943_1119 (size=12301) 2024-11-13T19:23:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-13T19:23:37,752 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:37,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-13T19:23:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T19:23:37,759 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:37,760 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:37,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525877841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525877842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525877842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T19:23:37,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T19:23:37,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:37,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:37,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T19:23:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T19:23:38,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/2ebdd3d3e81649038bcb3ddd5a54fab2 2024-11-13T19:23:38,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f0a31d2e19e74ed8bbb858555611e45f is 50, key is test_row_0/C:col10/1731525817194/Put/seqid=0 2024-11-13T19:23:38,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741944_1120 (size=12301) 2024-11-13T19:23:38,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T19:23:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525878351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525878351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525878351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,353 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T19:23:38,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T19:23:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:38,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:38,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=519 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f0a31d2e19e74ed8bbb858555611e45f 2024-11-13T19:23:38,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/72cf3a4705a748cf92f8a7ff9792a2e0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/72cf3a4705a748cf92f8a7ff9792a2e0 2024-11-13T19:23:38,511 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/72cf3a4705a748cf92f8a7ff9792a2e0, entries=150, sequenceid=519, filesize=12.0 K 2024-11-13T19:23:38,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/2ebdd3d3e81649038bcb3ddd5a54fab2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2ebdd3d3e81649038bcb3ddd5a54fab2 2024-11-13T19:23:38,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2ebdd3d3e81649038bcb3ddd5a54fab2, entries=150, sequenceid=519, filesize=12.0 K 2024-11-13T19:23:38,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f0a31d2e19e74ed8bbb858555611e45f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f0a31d2e19e74ed8bbb858555611e45f 2024-11-13T19:23:38,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f0a31d2e19e74ed8bbb858555611e45f, entries=150, sequenceid=519, filesize=12.0 K 2024-11-13T19:23:38,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 1330ms, sequenceid=519, compaction requested=true 2024-11-13T19:23:38,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:38,526 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:38,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:38,527 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:38,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:38,528 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:38,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:38,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:38,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:38,528 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:38,528 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,528 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/70fbf41bb4e849159e9515d24437dbde, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/67edc54150b5432396ca37da60ef410b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/72cf3a4705a748cf92f8a7ff9792a2e0] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=37.1 K 2024-11-13T19:23:38,529 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70fbf41bb4e849159e9515d24437dbde, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731525815302 2024-11-13T19:23:38,529 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67edc54150b5432396ca37da60ef410b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731525815323 2024-11-13T19:23:38,529 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:38,529 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:38,529 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,529 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72cf3a4705a748cf92f8a7ff9792a2e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731525816046 2024-11-13T19:23:38,529 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/287f2f2734b847009ac7c778c14f726d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2c9b5ee7120149e5881e0b7b270455b6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2ebdd3d3e81649038bcb3ddd5a54fab2] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=37.1 K 2024-11-13T19:23:38,530 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 287f2f2734b847009ac7c778c14f726d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731525815302 2024-11-13T19:23:38,530 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c9b5ee7120149e5881e0b7b270455b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731525815323 2024-11-13T19:23:38,531 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ebdd3d3e81649038bcb3ddd5a54fab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731525816046 2024-11-13T19:23:38,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-13T19:23:38,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,541 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-13T19:23:38,542 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:38,542 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/8300d6a9e43345c1bcea6506967e1bb3 is 50, key is test_row_0/A:col10/1731525817194/Put/seqid=0 2024-11-13T19:23:38,544 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:38,544 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/b20352a4df3642a68a51a29679a981fb is 50, key is test_row_0/B:col10/1731525817194/Put/seqid=0 2024-11-13T19:23:38,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/f7611b71663e439f8113bd1c21f18f3a is 50, key is test_row_0/A:col10/1731525817208/Put/seqid=0 2024-11-13T19:23:38,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741946_1122 (size=13527) 2024-11-13T19:23:38,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741947_1123 (size=12301) 2024-11-13T19:23:38,571 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/f7611b71663e439f8113bd1c21f18f3a 2024-11-13T19:23:38,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741945_1121 (size=13527) 2024-11-13T19:23:38,574 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/b20352a4df3642a68a51a29679a981fb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b20352a4df3642a68a51a29679a981fb 2024-11-13T19:23:38,581 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/8300d6a9e43345c1bcea6506967e1bb3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8300d6a9e43345c1bcea6506967e1bb3 2024-11-13T19:23:38,583 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into b20352a4df3642a68a51a29679a981fb(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:38,583 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:38,583 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525818527; duration=0sec 2024-11-13T19:23:38,584 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:38,584 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:38,584 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:38,586 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:38,586 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:38,586 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:38,586 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/69d05a97d6214594bf378893f373d010, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6ab3cd4bd40a40c3bd38196a7e044547, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f0a31d2e19e74ed8bbb858555611e45f] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=37.1 K 2024-11-13T19:23:38,587 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 69d05a97d6214594bf378893f373d010, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731525815302 2024-11-13T19:23:38,588 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab3cd4bd40a40c3bd38196a7e044547, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=488, earliestPutTs=1731525815323 2024-11-13T19:23:38,588 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f0a31d2e19e74ed8bbb858555611e45f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731525816046 2024-11-13T19:23:38,590 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into 8300d6a9e43345c1bcea6506967e1bb3(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:38,590 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:38,590 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525818526; duration=0sec 2024-11-13T19:23:38,590 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:38,590 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:38,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/9abe497c89cc4d458cfb761994675e04 is 50, key is test_row_0/B:col10/1731525817208/Put/seqid=0 2024-11-13T19:23:38,599 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#112 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:38,600 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/85644f5cdcc04a12be2bf527818c6039 is 50, key is test_row_0/C:col10/1731525817194/Put/seqid=0 2024-11-13T19:23:38,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741948_1124 (size=12301) 2024-11-13T19:23:38,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741949_1125 (size=13527) 2024-11-13T19:23:38,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. as already flushing 2024-11-13T19:23:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:38,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:38,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525878840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:38,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525878845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T19:23:38,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:38,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525878952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:38,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525878961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:39,006 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/9abe497c89cc4d458cfb761994675e04 2024-11-13T19:23:39,015 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/85644f5cdcc04a12be2bf527818c6039 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/85644f5cdcc04a12be2bf527818c6039 2024-11-13T19:23:39,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/33dc077f35cd4e9cad26d09146327147 is 50, key is test_row_0/C:col10/1731525817208/Put/seqid=0 2024-11-13T19:23:39,021 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into 85644f5cdcc04a12be2bf527818c6039(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:39,021 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:39,021 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525818528; duration=0sec 2024-11-13T19:23:39,022 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:39,022 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:39,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741950_1126 (size=12301) 2024-11-13T19:23:39,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57172 deadline: 1731525879162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:39,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:39,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57144 deadline: 1731525879172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,175 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:39,257 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-13T19:23:39,258 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:39,259 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-13T19:23:39,259 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:39,260 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-13T19:23:39,260 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:39,261 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-13T19:23:39,262 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:39,262 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,262 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,263 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,263 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,263 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,263 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,263 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:39,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:39,263 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:39,266 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:39,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57166 deadline: 1731525879362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57132 deadline: 1731525879362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57140 deadline: 1731525879363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:39,364 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in 1334c8616b794fd8c4fb42c4bfbc0d50 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:39,364 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 1334c8616b794fd8c4fb42c4bfbc0d50 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:39,364 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in 1334c8616b794fd8c4fb42c4bfbc0d50 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:23:39,427 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=528 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/33dc077f35cd4e9cad26d09146327147 2024-11-13T19:23:39,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/f7611b71663e439f8113bd1c21f18f3a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f7611b71663e439f8113bd1c21f18f3a 2024-11-13T19:23:39,442 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f7611b71663e439f8113bd1c21f18f3a, entries=150, sequenceid=528, filesize=12.0 K 2024-11-13T19:23:39,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/9abe497c89cc4d458cfb761994675e04 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/9abe497c89cc4d458cfb761994675e04 2024-11-13T19:23:39,449 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/9abe497c89cc4d458cfb761994675e04, entries=150, sequenceid=528, filesize=12.0 K 2024-11-13T19:23:39,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/33dc077f35cd4e9cad26d09146327147 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/33dc077f35cd4e9cad26d09146327147 2024-11-13T19:23:39,456 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/33dc077f35cd4e9cad26d09146327147, entries=150, sequenceid=528, filesize=12.0 K 2024-11-13T19:23:39,457 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 915ms, sequenceid=528, compaction requested=false 2024-11-13T19:23:39,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:39,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:39,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-13T19:23:39,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-13T19:23:39,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-13T19:23:39,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6980 sec 2024-11-13T19:23:39,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.7070 sec 2024-11-13T19:23:39,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:39,473 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:39,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T19:23:39,473 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-13T19:23:39,473 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:39,473 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,473 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,473 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:39,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/e543bc75ae2849b287e09eaf5be09355 is 50, key is test_row_0/A:col10/1731525818838/Put/seqid=0 2024-11-13T19:23:39,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741951_1127 (size=12301) 2024-11-13T19:23:39,484 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-13T19:23:39,484 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:39,484 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,484 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:39,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:39,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/e543bc75ae2849b287e09eaf5be09355 2024-11-13T19:23:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-13T19:23:39,912 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:39,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/7d3a5f7a1cc24daaa7fa50bb9dafba95 is 50, key is test_row_0/B:col10/1731525818838/Put/seqid=0 2024-11-13T19:23:39,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741952_1128 (size=12301) 2024-11-13T19:23:40,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/7d3a5f7a1cc24daaa7fa50bb9dafba95 2024-11-13T19:23:40,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f1a0843b4b6344229e532fca2bf676db is 50, key is test_row_0/C:col10/1731525818838/Put/seqid=0 2024-11-13T19:23:40,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741953_1129 (size=12301) 2024-11-13T19:23:40,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f1a0843b4b6344229e532fca2bf676db 2024-11-13T19:23:40,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/e543bc75ae2849b287e09eaf5be09355 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/e543bc75ae2849b287e09eaf5be09355 2024-11-13T19:23:40,751 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:23:40,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/e543bc75ae2849b287e09eaf5be09355, entries=150, sequenceid=559, filesize=12.0 K 2024-11-13T19:23:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/7d3a5f7a1cc24daaa7fa50bb9dafba95 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/7d3a5f7a1cc24daaa7fa50bb9dafba95 2024-11-13T19:23:40,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/7d3a5f7a1cc24daaa7fa50bb9dafba95, entries=150, sequenceid=559, filesize=12.0 K 2024-11-13T19:23:40,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f1a0843b4b6344229e532fca2bf676db as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f1a0843b4b6344229e532fca2bf676db 2024-11-13T19:23:40,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f1a0843b4b6344229e532fca2bf676db, entries=150, sequenceid=559, filesize=12.0 K 2024-11-13T19:23:40,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=6.71 KB/6870 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 1301ms, sequenceid=559, compaction requested=true 2024-11-13T19:23:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:40,774 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:40,774 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1334c8616b794fd8c4fb42c4bfbc0d50:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:40,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:40,775 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:40,775 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:40,775 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/A is initiating minor compaction (all files) 2024-11-13T19:23:40,775 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/B is initiating minor compaction (all files) 2024-11-13T19:23:40,775 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/A in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:40,775 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/B in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:40,776 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8300d6a9e43345c1bcea6506967e1bb3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f7611b71663e439f8113bd1c21f18f3a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/e543bc75ae2849b287e09eaf5be09355] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=37.2 K 2024-11-13T19:23:40,776 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b20352a4df3642a68a51a29679a981fb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/9abe497c89cc4d458cfb761994675e04, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/7d3a5f7a1cc24daaa7fa50bb9dafba95] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=37.2 K 2024-11-13T19:23:40,776 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8300d6a9e43345c1bcea6506967e1bb3, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731525816046 2024-11-13T19:23:40,776 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting b20352a4df3642a68a51a29679a981fb, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731525816046 2024-11-13T19:23:40,777 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7611b71663e439f8113bd1c21f18f3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1731525817201 2024-11-13T19:23:40,777 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9abe497c89cc4d458cfb761994675e04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1731525817201 2024-11-13T19:23:40,777 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e543bc75ae2849b287e09eaf5be09355, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731525818838 2024-11-13T19:23:40,777 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d3a5f7a1cc24daaa7fa50bb9dafba95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731525818838 2024-11-13T19:23:40,787 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#B#compaction#118 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:40,787 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#A#compaction#117 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:40,787 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f41812af39d5413f95af788d6501c330 is 50, key is test_row_0/B:col10/1731525818838/Put/seqid=0 2024-11-13T19:23:40,787 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b49f53ae568148fe91fee3f5427346b6 is 50, key is test_row_0/A:col10/1731525818838/Put/seqid=0 2024-11-13T19:23:40,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741955_1131 (size=13629) 2024-11-13T19:23:40,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741954_1130 (size=13629) 2024-11-13T19:23:41,200 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/b49f53ae568148fe91fee3f5427346b6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b49f53ae568148fe91fee3f5427346b6 2024-11-13T19:23:41,203 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/f41812af39d5413f95af788d6501c330 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f41812af39d5413f95af788d6501c330 2024-11-13T19:23:41,207 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/A of 1334c8616b794fd8c4fb42c4bfbc0d50 into b49f53ae568148fe91fee3f5427346b6(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:41,207 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:41,207 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/A, priority=13, startTime=1731525820774; duration=0sec 2024-11-13T19:23:41,207 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:41,207 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:A 2024-11-13T19:23:41,208 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:41,209 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:41,209 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 1334c8616b794fd8c4fb42c4bfbc0d50/C is initiating minor compaction (all files) 2024-11-13T19:23:41,209 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1334c8616b794fd8c4fb42c4bfbc0d50/C in TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:41,209 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/85644f5cdcc04a12be2bf527818c6039, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/33dc077f35cd4e9cad26d09146327147, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f1a0843b4b6344229e532fca2bf676db] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp, totalSize=37.2 K 2024-11-13T19:23:41,209 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/B of 1334c8616b794fd8c4fb42c4bfbc0d50 into f41812af39d5413f95af788d6501c330(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:41,209 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:41,209 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/B, priority=13, startTime=1731525820774; duration=0sec 2024-11-13T19:23:41,210 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:41,210 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85644f5cdcc04a12be2bf527818c6039, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=519, earliestPutTs=1731525816046 2024-11-13T19:23:41,210 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:B 2024-11-13T19:23:41,210 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33dc077f35cd4e9cad26d09146327147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=528, earliestPutTs=1731525817201 2024-11-13T19:23:41,210 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a0843b4b6344229e532fca2bf676db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731525818838 2024-11-13T19:23:41,219 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1334c8616b794fd8c4fb42c4bfbc0d50#C#compaction#119 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:41,219 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5e52ff24b377468e9e293a47687a8e8a is 50, key is test_row_0/C:col10/1731525818838/Put/seqid=0 2024-11-13T19:23:41,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741956_1132 (size=13629) 2024-11-13T19:23:41,384 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-13T19:23:41,384 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:41,384 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,384 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:41,393 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-13T19:23:41,393 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-13T19:23:41,393 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:41,393 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:23:41,393 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,393 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,393 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,393 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:41,393 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:23:41,393 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 92 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3273 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3192 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1512 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4536 rows 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1521 2024-11-13T19:23:41,394 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4553 rows 2024-11-13T19:23:41,394 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:23:41,394 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:23:41,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:41,394 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:23:41,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:23:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:41,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T19:23:41,405 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525821405"}]},"ts":"1731525821405"} 2024-11-13T19:23:41,407 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:23:41,407 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:23:41,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:23:41,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, UNASSIGN}] 2024-11-13T19:23:41,414 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, UNASSIGN 2024-11-13T19:23:41,416 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1334c8616b794fd8c4fb42c4bfbc0d50, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:41,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, UNASSIGN because future has completed 2024-11-13T19:23:41,418 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:23:41,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:23:41,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T19:23:41,575 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:41,575 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:23:41,575 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 1334c8616b794fd8c4fb42c4bfbc0d50, disabling compactions & flushes 2024-11-13T19:23:41,576 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:41,630 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/5e52ff24b377468e9e293a47687a8e8a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5e52ff24b377468e9e293a47687a8e8a 2024-11-13T19:23:41,638 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1334c8616b794fd8c4fb42c4bfbc0d50/C of 1334c8616b794fd8c4fb42c4bfbc0d50 into 5e52ff24b377468e9e293a47687a8e8a(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:41,638 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1334c8616b794fd8c4fb42c4bfbc0d50: 2024-11-13T19:23:41,638 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:41,638 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., storeName=1334c8616b794fd8c4fb42c4bfbc0d50/C, priority=13, startTime=1731525820774; duration=0sec 2024-11-13T19:23:41,638 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:41,638 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:41,638 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. after waiting 0 ms 2024-11-13T19:23:41,638 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1334c8616b794fd8c4fb42c4bfbc0d50:C 2024-11-13T19:23:41,638 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:41,638 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 1334c8616b794fd8c4fb42c4bfbc0d50 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T19:23:41,642 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/44a06d9038b24f9b840655a92109b620 is 50, key is test_row_0/A:col10/1731525821392/Put/seqid=0 2024-11-13T19:23:41,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741957_1133 (size=9857) 2024-11-13T19:23:41,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T19:23:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T19:23:42,047 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/44a06d9038b24f9b840655a92109b620 2024-11-13T19:23:42,054 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/948d34a82ed44ac8b5f6bfd55930af5c is 50, key is test_row_0/B:col10/1731525821392/Put/seqid=0 2024-11-13T19:23:42,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741958_1134 (size=9857) 2024-11-13T19:23:42,458 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/948d34a82ed44ac8b5f6bfd55930af5c 2024-11-13T19:23:42,466 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f844f579a4a04189a102d47b8f0ed5c0 is 50, key is test_row_0/C:col10/1731525821392/Put/seqid=0 2024-11-13T19:23:42,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741959_1135 (size=9857) 2024-11-13T19:23:42,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T19:23:42,873 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f844f579a4a04189a102d47b8f0ed5c0 2024-11-13T19:23:42,882 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/A/44a06d9038b24f9b840655a92109b620 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/44a06d9038b24f9b840655a92109b620 2024-11-13T19:23:42,888 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/44a06d9038b24f9b840655a92109b620, entries=100, sequenceid=569, filesize=9.6 K 2024-11-13T19:23:42,890 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/B/948d34a82ed44ac8b5f6bfd55930af5c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/948d34a82ed44ac8b5f6bfd55930af5c 2024-11-13T19:23:42,895 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/948d34a82ed44ac8b5f6bfd55930af5c, entries=100, sequenceid=569, filesize=9.6 K 2024-11-13T19:23:42,896 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/.tmp/C/f844f579a4a04189a102d47b8f0ed5c0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f844f579a4a04189a102d47b8f0ed5c0 2024-11-13T19:23:42,901 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f844f579a4a04189a102d47b8f0ed5c0, entries=100, sequenceid=569, filesize=9.6 K 2024-11-13T19:23:42,902 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 1264ms, sequenceid=569, compaction requested=false 2024-11-13T19:23:42,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b200021f699d446db30889ef1c116215, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3ec2625cf604580ab6e7fcc269eab24, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/6f1828abbcd847a195cfd8acfd6d7967, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/1e8feef1d1ab4131a9fbe5c33beb65b8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8720f776b6464cb180da5e50a5fb7211, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3e090fa86394f5d878609c5cff3a808, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/493b0634adff4959b7458c38a1006dc8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/02885f2f61094b06967cb7276ec4ff6b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b7bfd88eef3a4defada130cb9df7accb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/473756de85764cf2a8bcd260042069bb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/12f0b28564734dd18283b54bfa847e43, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/68cf528a00f44cc9a4039eb9617387be, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/4a8c45cd38b748e4835154fb3e19629f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/ede83bb5010f41c3866e93415c08b893, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/607bf08ede824569bc6c39e4dcc87f7c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0da35f1877f449f5b7f50da4a791ea09, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f6d74fa6b0b0453c900ec5381d0c6195, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/bbcb0707dbe242e5979b1c672aa17996, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/914b49f937a3475cae36b5522f3aa6b3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/59748f8195e14514b4a1a3a154208089, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c42e751f22824c888db480ba1fdeadae, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/cef69c7e58784085b22b45df6ea19138, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/2812e53da08b462d85169ad6d1c8c476, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0a5da6f687c6400387dfa66d3cfc73d9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5c1024bfbb944a93b33b2757c4fd6465, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c6d77f39ed60404aab44696d98aeeeca, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/523afef9f5674cfa8e159dfbbcffbd54, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5cdb867ae8904d539ec7ef6fb69d9a78, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c1bad009ace14e32a681fc1bac6f36d0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/13b0b0ef816242c597b232ee6f1ab559, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b4493639941140c19e06b4e84a0cff63, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/3e8fbdf911cf4a529ae2acb80038271c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5bf89365abec4d058c935f9b5cbc1619, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/70fbf41bb4e849159e9515d24437dbde, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/67edc54150b5432396ca37da60ef410b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8300d6a9e43345c1bcea6506967e1bb3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/72cf3a4705a748cf92f8a7ff9792a2e0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f7611b71663e439f8113bd1c21f18f3a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/e543bc75ae2849b287e09eaf5be09355] to archive 2024-11-13T19:23:42,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:23:42,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b200021f699d446db30889ef1c116215 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b200021f699d446db30889ef1c116215 2024-11-13T19:23:42,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3ec2625cf604580ab6e7fcc269eab24 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3ec2625cf604580ab6e7fcc269eab24 2024-11-13T19:23:42,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/6f1828abbcd847a195cfd8acfd6d7967 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/6f1828abbcd847a195cfd8acfd6d7967 2024-11-13T19:23:42,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/1e8feef1d1ab4131a9fbe5c33beb65b8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/1e8feef1d1ab4131a9fbe5c33beb65b8 2024-11-13T19:23:42,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8720f776b6464cb180da5e50a5fb7211 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8720f776b6464cb180da5e50a5fb7211 2024-11-13T19:23:42,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3e090fa86394f5d878609c5cff3a808 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b3e090fa86394f5d878609c5cff3a808 2024-11-13T19:23:42,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/493b0634adff4959b7458c38a1006dc8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/493b0634adff4959b7458c38a1006dc8 2024-11-13T19:23:42,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/02885f2f61094b06967cb7276ec4ff6b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/02885f2f61094b06967cb7276ec4ff6b 2024-11-13T19:23:42,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b7bfd88eef3a4defada130cb9df7accb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b7bfd88eef3a4defada130cb9df7accb 2024-11-13T19:23:42,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/473756de85764cf2a8bcd260042069bb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/473756de85764cf2a8bcd260042069bb 2024-11-13T19:23:42,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/12f0b28564734dd18283b54bfa847e43 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/12f0b28564734dd18283b54bfa847e43 2024-11-13T19:23:42,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/68cf528a00f44cc9a4039eb9617387be to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/68cf528a00f44cc9a4039eb9617387be 2024-11-13T19:23:42,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/4a8c45cd38b748e4835154fb3e19629f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/4a8c45cd38b748e4835154fb3e19629f 2024-11-13T19:23:42,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/ede83bb5010f41c3866e93415c08b893 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/ede83bb5010f41c3866e93415c08b893 2024-11-13T19:23:42,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/607bf08ede824569bc6c39e4dcc87f7c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/607bf08ede824569bc6c39e4dcc87f7c 2024-11-13T19:23:42,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0da35f1877f449f5b7f50da4a791ea09 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0da35f1877f449f5b7f50da4a791ea09 2024-11-13T19:23:42,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f6d74fa6b0b0453c900ec5381d0c6195 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f6d74fa6b0b0453c900ec5381d0c6195 2024-11-13T19:23:42,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/bbcb0707dbe242e5979b1c672aa17996 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/bbcb0707dbe242e5979b1c672aa17996 2024-11-13T19:23:42,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/914b49f937a3475cae36b5522f3aa6b3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/914b49f937a3475cae36b5522f3aa6b3 2024-11-13T19:23:42,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/59748f8195e14514b4a1a3a154208089 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/59748f8195e14514b4a1a3a154208089 2024-11-13T19:23:42,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c42e751f22824c888db480ba1fdeadae to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c42e751f22824c888db480ba1fdeadae 2024-11-13T19:23:42,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/cef69c7e58784085b22b45df6ea19138 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/cef69c7e58784085b22b45df6ea19138 2024-11-13T19:23:42,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/2812e53da08b462d85169ad6d1c8c476 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/2812e53da08b462d85169ad6d1c8c476 2024-11-13T19:23:42,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0a5da6f687c6400387dfa66d3cfc73d9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/0a5da6f687c6400387dfa66d3cfc73d9 2024-11-13T19:23:42,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5c1024bfbb944a93b33b2757c4fd6465 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5c1024bfbb944a93b33b2757c4fd6465 2024-11-13T19:23:42,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c6d77f39ed60404aab44696d98aeeeca to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c6d77f39ed60404aab44696d98aeeeca 2024-11-13T19:23:42,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/523afef9f5674cfa8e159dfbbcffbd54 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/523afef9f5674cfa8e159dfbbcffbd54 2024-11-13T19:23:42,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5cdb867ae8904d539ec7ef6fb69d9a78 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5cdb867ae8904d539ec7ef6fb69d9a78 2024-11-13T19:23:42,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c1bad009ace14e32a681fc1bac6f36d0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/c1bad009ace14e32a681fc1bac6f36d0 2024-11-13T19:23:42,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/13b0b0ef816242c597b232ee6f1ab559 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/13b0b0ef816242c597b232ee6f1ab559 2024-11-13T19:23:42,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b4493639941140c19e06b4e84a0cff63 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b4493639941140c19e06b4e84a0cff63 2024-11-13T19:23:42,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/3e8fbdf911cf4a529ae2acb80038271c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/3e8fbdf911cf4a529ae2acb80038271c 2024-11-13T19:23:42,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5bf89365abec4d058c935f9b5cbc1619 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/5bf89365abec4d058c935f9b5cbc1619 2024-11-13T19:23:42,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/70fbf41bb4e849159e9515d24437dbde to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/70fbf41bb4e849159e9515d24437dbde 2024-11-13T19:23:42,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/67edc54150b5432396ca37da60ef410b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/67edc54150b5432396ca37da60ef410b 2024-11-13T19:23:42,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8300d6a9e43345c1bcea6506967e1bb3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/8300d6a9e43345c1bcea6506967e1bb3 2024-11-13T19:23:42,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/72cf3a4705a748cf92f8a7ff9792a2e0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/72cf3a4705a748cf92f8a7ff9792a2e0 2024-11-13T19:23:42,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f7611b71663e439f8113bd1c21f18f3a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/f7611b71663e439f8113bd1c21f18f3a 2024-11-13T19:23:42,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/e543bc75ae2849b287e09eaf5be09355 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/e543bc75ae2849b287e09eaf5be09355 2024-11-13T19:23:42,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/eaaadb1a698c46e59ed2530461f32856, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/00bcec48e0ad425d8021967d10547a13, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/02f607160d0847b283c3770f107629b3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/d7c5a74a7a004acfb8a0be7e3f16e41d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/c82eac95641544029fd56d278ed407a1, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a558395291fb4261a4a0d3deb7b70ccf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f241b5121987497d93ebc315a7f5203f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/caa09aab9cb2408caff5354022c91e6e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/309387167d434fcfa97ae1711a92b1ee, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a9ef1c2f0f6c4970bcf0b2e8883f892e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/4987763020914de89274ba6aaaa69ec3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/07d96ab1ae9e46ec9da0b261694884b4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/147b8e70b3d344898e74ff323c42d0f7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e0e44c96c77348c3ae2fe2b83ce6ef37, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5288daab5ef14f3f81471f1fca863c66, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/15792bce0287490baa5194a410dacdfe, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e282378b24de4466992a26696d694b21, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e9c74a231ef54fc5953fc851d196c814, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/8f7972812fae4bdba83248cf882db643, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/0e36d305d58146d6908aa75901eb3f68, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f492dfaf8ae74dc6bc257368a575784e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5ed76d359b034ccca41ceae08910fd0d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b5576ff45d564894abfbc99893065547, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/512a02cc63cf480aab5520dc53bff82d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/789ddb0f8fbb46b8906a680cc13bd8b2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/924101d559c8400ebd195e453c3aba83, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/40a970596c354051990958d0b012f0da, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5a312603fd5c4ceba6ebb6dde999896f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/509f02562e514ebe9bad8f442d856bfa, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/06a1cebb7a454ab8af1aea3777a6811d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/660f86d396c04de3a4d4e5963b03ef6a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/90e5a8a3dc72478bad0a6daed96cf1b5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/287f2f2734b847009ac7c778c14f726d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e7769e6f46b34142a2d1c6fa4cde539d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2c9b5ee7120149e5881e0b7b270455b6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b20352a4df3642a68a51a29679a981fb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2ebdd3d3e81649038bcb3ddd5a54fab2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/9abe497c89cc4d458cfb761994675e04, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/7d3a5f7a1cc24daaa7fa50bb9dafba95] to archive 2024-11-13T19:23:42,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:23:42,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/eaaadb1a698c46e59ed2530461f32856 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/eaaadb1a698c46e59ed2530461f32856 2024-11-13T19:23:42,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/00bcec48e0ad425d8021967d10547a13 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/00bcec48e0ad425d8021967d10547a13 2024-11-13T19:23:42,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/02f607160d0847b283c3770f107629b3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/02f607160d0847b283c3770f107629b3 2024-11-13T19:23:42,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/d7c5a74a7a004acfb8a0be7e3f16e41d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/d7c5a74a7a004acfb8a0be7e3f16e41d 2024-11-13T19:23:42,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/c82eac95641544029fd56d278ed407a1 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/c82eac95641544029fd56d278ed407a1 2024-11-13T19:23:42,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a558395291fb4261a4a0d3deb7b70ccf to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a558395291fb4261a4a0d3deb7b70ccf 2024-11-13T19:23:42,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f241b5121987497d93ebc315a7f5203f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f241b5121987497d93ebc315a7f5203f 2024-11-13T19:23:42,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/caa09aab9cb2408caff5354022c91e6e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/caa09aab9cb2408caff5354022c91e6e 2024-11-13T19:23:42,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/309387167d434fcfa97ae1711a92b1ee to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/309387167d434fcfa97ae1711a92b1ee 2024-11-13T19:23:42,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a9ef1c2f0f6c4970bcf0b2e8883f892e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/a9ef1c2f0f6c4970bcf0b2e8883f892e 2024-11-13T19:23:42,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/4987763020914de89274ba6aaaa69ec3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/4987763020914de89274ba6aaaa69ec3 2024-11-13T19:23:42,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/07d96ab1ae9e46ec9da0b261694884b4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/07d96ab1ae9e46ec9da0b261694884b4 2024-11-13T19:23:42,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/147b8e70b3d344898e74ff323c42d0f7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/147b8e70b3d344898e74ff323c42d0f7 2024-11-13T19:23:43,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e0e44c96c77348c3ae2fe2b83ce6ef37 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e0e44c96c77348c3ae2fe2b83ce6ef37 2024-11-13T19:23:43,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5288daab5ef14f3f81471f1fca863c66 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5288daab5ef14f3f81471f1fca863c66 2024-11-13T19:23:43,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/15792bce0287490baa5194a410dacdfe to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/15792bce0287490baa5194a410dacdfe 2024-11-13T19:23:43,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e282378b24de4466992a26696d694b21 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e282378b24de4466992a26696d694b21 2024-11-13T19:23:43,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e9c74a231ef54fc5953fc851d196c814 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e9c74a231ef54fc5953fc851d196c814 2024-11-13T19:23:43,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/8f7972812fae4bdba83248cf882db643 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/8f7972812fae4bdba83248cf882db643 2024-11-13T19:23:43,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/0e36d305d58146d6908aa75901eb3f68 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/0e36d305d58146d6908aa75901eb3f68 2024-11-13T19:23:43,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f492dfaf8ae74dc6bc257368a575784e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f492dfaf8ae74dc6bc257368a575784e 2024-11-13T19:23:43,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5ed76d359b034ccca41ceae08910fd0d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5ed76d359b034ccca41ceae08910fd0d 2024-11-13T19:23:43,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b5576ff45d564894abfbc99893065547 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b5576ff45d564894abfbc99893065547 2024-11-13T19:23:43,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/512a02cc63cf480aab5520dc53bff82d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/512a02cc63cf480aab5520dc53bff82d 2024-11-13T19:23:43,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/789ddb0f8fbb46b8906a680cc13bd8b2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/789ddb0f8fbb46b8906a680cc13bd8b2 2024-11-13T19:23:43,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/924101d559c8400ebd195e453c3aba83 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/924101d559c8400ebd195e453c3aba83 2024-11-13T19:23:43,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/40a970596c354051990958d0b012f0da to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/40a970596c354051990958d0b012f0da 2024-11-13T19:23:43,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5a312603fd5c4ceba6ebb6dde999896f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/5a312603fd5c4ceba6ebb6dde999896f 2024-11-13T19:23:43,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/509f02562e514ebe9bad8f442d856bfa to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/509f02562e514ebe9bad8f442d856bfa 2024-11-13T19:23:43,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/06a1cebb7a454ab8af1aea3777a6811d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/06a1cebb7a454ab8af1aea3777a6811d 2024-11-13T19:23:43,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/660f86d396c04de3a4d4e5963b03ef6a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/660f86d396c04de3a4d4e5963b03ef6a 2024-11-13T19:23:43,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/90e5a8a3dc72478bad0a6daed96cf1b5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/90e5a8a3dc72478bad0a6daed96cf1b5 2024-11-13T19:23:43,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/287f2f2734b847009ac7c778c14f726d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/287f2f2734b847009ac7c778c14f726d 2024-11-13T19:23:43,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e7769e6f46b34142a2d1c6fa4cde539d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/e7769e6f46b34142a2d1c6fa4cde539d 2024-11-13T19:23:43,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2c9b5ee7120149e5881e0b7b270455b6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2c9b5ee7120149e5881e0b7b270455b6 2024-11-13T19:23:43,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b20352a4df3642a68a51a29679a981fb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/b20352a4df3642a68a51a29679a981fb 2024-11-13T19:23:43,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2ebdd3d3e81649038bcb3ddd5a54fab2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/2ebdd3d3e81649038bcb3ddd5a54fab2 2024-11-13T19:23:43,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/9abe497c89cc4d458cfb761994675e04 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/9abe497c89cc4d458cfb761994675e04 2024-11-13T19:23:43,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/7d3a5f7a1cc24daaa7fa50bb9dafba95 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/7d3a5f7a1cc24daaa7fa50bb9dafba95 2024-11-13T19:23:43,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/36bdabf2e6674fb199b8e93334550586, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/167c559a5adb47c9ad1b28e00bcdbaf0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b0cf850e7d54477b9990e1ff649693c4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5d1b1c1bd61142a88172308f57e5f087, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/07211a5a88864e3bb49722652a13aafa, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bfebe18b7e9147ffb9cfc258a96ed8af, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/21178898cfc8427eba1b3aa56387ee8b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/25eacbb5bd4749459a3ed836dbf79679, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/fdd1a76df0b4417dbdb8ab78bf4feeb9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/970dde99b4d74ae0b96aae25cd141eb7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a23cc797ab98410ab0e56c6091d1d66e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/4aa53fefd93a4eaabffbe0344ff3e866, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b5d6e2589086416893a7123681308e7b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6fe5dc32a8aa46b29f061bf22240c1b5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e6353d4687824eaa91ca3f30332065e9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/17e30f26bc3c4f5484416dc791c68639, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a56d8fbc046c4223bdbc673329466476, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a79ceb6f2c744b58807fe746473c6e6d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5ca61fcc90b849f885f24d9ea343c022, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5cbb4b75d6aa4b3987e656ede5a3e642, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b75456a96653436a8215934e57f95163, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a5f89709de7e4210b4c5ec1b5525cd43, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/d0a4585583604cd0911869ce64c3295b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/348daf40132c47ffb2a57ee22cb45e00, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e831f1f0edf42bb8dcadc0a5aacff95, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a3eb898747bf4f249e29bb84afd990a4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e907d55608994c988f574862ef0e7d11, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/09cf06f25fd24b60ae4f4b9a5b31f9c7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bd8432bba5d4450ab525035ac4d2ba8a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/c210d2d6821b42a0b3e1d2ba04220c16, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/69d05a97d6214594bf378893f373d010, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/2dd74e80d193434c85179aba88a871d4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6ab3cd4bd40a40c3bd38196a7e044547, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/85644f5cdcc04a12be2bf527818c6039, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f0a31d2e19e74ed8bbb858555611e45f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/33dc077f35cd4e9cad26d09146327147, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f1a0843b4b6344229e532fca2bf676db] to archive 2024-11-13T19:23:43,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:23:43,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/36bdabf2e6674fb199b8e93334550586 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/36bdabf2e6674fb199b8e93334550586 2024-11-13T19:23:43,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/167c559a5adb47c9ad1b28e00bcdbaf0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/167c559a5adb47c9ad1b28e00bcdbaf0 2024-11-13T19:23:43,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b0cf850e7d54477b9990e1ff649693c4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b0cf850e7d54477b9990e1ff649693c4 2024-11-13T19:23:43,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5d1b1c1bd61142a88172308f57e5f087 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5d1b1c1bd61142a88172308f57e5f087 2024-11-13T19:23:43,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/07211a5a88864e3bb49722652a13aafa to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/07211a5a88864e3bb49722652a13aafa 2024-11-13T19:23:43,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bfebe18b7e9147ffb9cfc258a96ed8af to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bfebe18b7e9147ffb9cfc258a96ed8af 2024-11-13T19:23:43,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/21178898cfc8427eba1b3aa56387ee8b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/21178898cfc8427eba1b3aa56387ee8b 2024-11-13T19:23:43,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/25eacbb5bd4749459a3ed836dbf79679 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/25eacbb5bd4749459a3ed836dbf79679 2024-11-13T19:23:43,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/fdd1a76df0b4417dbdb8ab78bf4feeb9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/fdd1a76df0b4417dbdb8ab78bf4feeb9 2024-11-13T19:23:43,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/970dde99b4d74ae0b96aae25cd141eb7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/970dde99b4d74ae0b96aae25cd141eb7 2024-11-13T19:23:43,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a23cc797ab98410ab0e56c6091d1d66e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a23cc797ab98410ab0e56c6091d1d66e 2024-11-13T19:23:43,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/4aa53fefd93a4eaabffbe0344ff3e866 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/4aa53fefd93a4eaabffbe0344ff3e866 2024-11-13T19:23:43,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b5d6e2589086416893a7123681308e7b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b5d6e2589086416893a7123681308e7b 2024-11-13T19:23:43,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6fe5dc32a8aa46b29f061bf22240c1b5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6fe5dc32a8aa46b29f061bf22240c1b5 2024-11-13T19:23:43,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e6353d4687824eaa91ca3f30332065e9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e6353d4687824eaa91ca3f30332065e9 2024-11-13T19:23:43,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/17e30f26bc3c4f5484416dc791c68639 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/17e30f26bc3c4f5484416dc791c68639 2024-11-13T19:23:43,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a56d8fbc046c4223bdbc673329466476 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a56d8fbc046c4223bdbc673329466476 2024-11-13T19:23:43,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a79ceb6f2c744b58807fe746473c6e6d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a79ceb6f2c744b58807fe746473c6e6d 2024-11-13T19:23:43,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5ca61fcc90b849f885f24d9ea343c022 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5ca61fcc90b849f885f24d9ea343c022 2024-11-13T19:23:43,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5cbb4b75d6aa4b3987e656ede5a3e642 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5cbb4b75d6aa4b3987e656ede5a3e642 2024-11-13T19:23:43,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b75456a96653436a8215934e57f95163 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/b75456a96653436a8215934e57f95163 2024-11-13T19:23:43,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a5f89709de7e4210b4c5ec1b5525cd43 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a5f89709de7e4210b4c5ec1b5525cd43 2024-11-13T19:23:43,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f9ed0ecea1ef475c8ec46f9ad5c6bf14 2024-11-13T19:23:43,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/d0a4585583604cd0911869ce64c3295b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/d0a4585583604cd0911869ce64c3295b 2024-11-13T19:23:43,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/348daf40132c47ffb2a57ee22cb45e00 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/348daf40132c47ffb2a57ee22cb45e00 2024-11-13T19:23:43,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e831f1f0edf42bb8dcadc0a5aacff95 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e831f1f0edf42bb8dcadc0a5aacff95 2024-11-13T19:23:43,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a3eb898747bf4f249e29bb84afd990a4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/a3eb898747bf4f249e29bb84afd990a4 2024-11-13T19:23:43,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e907d55608994c988f574862ef0e7d11 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/e907d55608994c988f574862ef0e7d11 2024-11-13T19:23:43,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/09cf06f25fd24b60ae4f4b9a5b31f9c7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/09cf06f25fd24b60ae4f4b9a5b31f9c7 2024-11-13T19:23:43,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bd8432bba5d4450ab525035ac4d2ba8a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/bd8432bba5d4450ab525035ac4d2ba8a 2024-11-13T19:23:43,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/9e097ffbad8a4e8a8d5b6d9f75c77ff6 2024-11-13T19:23:43,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/c210d2d6821b42a0b3e1d2ba04220c16 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/c210d2d6821b42a0b3e1d2ba04220c16 2024-11-13T19:23:43,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/69d05a97d6214594bf378893f373d010 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/69d05a97d6214594bf378893f373d010 2024-11-13T19:23:43,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/2dd74e80d193434c85179aba88a871d4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/2dd74e80d193434c85179aba88a871d4 2024-11-13T19:23:43,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6ab3cd4bd40a40c3bd38196a7e044547 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/6ab3cd4bd40a40c3bd38196a7e044547 2024-11-13T19:23:43,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/85644f5cdcc04a12be2bf527818c6039 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/85644f5cdcc04a12be2bf527818c6039 2024-11-13T19:23:43,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f0a31d2e19e74ed8bbb858555611e45f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f0a31d2e19e74ed8bbb858555611e45f 2024-11-13T19:23:43,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/33dc077f35cd4e9cad26d09146327147 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/33dc077f35cd4e9cad26d09146327147 2024-11-13T19:23:43,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f1a0843b4b6344229e532fca2bf676db to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f1a0843b4b6344229e532fca2bf676db 2024-11-13T19:23:43,079 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/recovered.edits/572.seqid, newMaxSeqId=572, maxSeqId=1 2024-11-13T19:23:43,080 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. 2024-11-13T19:23:43,080 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 1334c8616b794fd8c4fb42c4bfbc0d50: Waiting for close lock at 1731525821575Running coprocessor pre-close hooks at 1731525821575Disabling compacts and flushes for region at 1731525821575Disabling writes for close at 1731525821638 (+63 ms)Obtaining lock to block concurrent updates at 1731525821638Preparing flush snapshotting stores in 1334c8616b794fd8c4fb42c4bfbc0d50 at 1731525821638Finished memstore snapshotting TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731525821639 (+1 ms)Flushing stores of TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50. at 1731525821639Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/A: creating writer at 1731525821639Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/A: appending metadata at 1731525821642 (+3 ms)Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/A: closing flushed file at 1731525821642Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/B: creating writer at 1731525822051 (+409 ms)Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/B: appending metadata at 1731525822053 (+2 ms)Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/B: closing flushed file at 1731525822053Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/C: creating writer at 1731525822463 (+410 ms)Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/C: appending metadata at 1731525822465 (+2 ms)Flushing 1334c8616b794fd8c4fb42c4bfbc0d50/C: closing flushed file at 1731525822466 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a42912e: reopening flushed file at 1731525822881 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70a13cf0: reopening flushed file at 1731525822889 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10f716ce: reopening flushed file at 1731525822895 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 1334c8616b794fd8c4fb42c4bfbc0d50 in 1264ms, sequenceid=569, compaction requested=false at 1731525822902 (+7 ms)Writing region close event to WAL at 1731525823076 (+174 ms)Running coprocessor post-close hooks at 1731525823080 (+4 ms)Closed at 1731525823080 2024-11-13T19:23:43,082 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:43,083 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1334c8616b794fd8c4fb42c4bfbc0d50, regionState=CLOSED 2024-11-13T19:23:43,085 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:23:43,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-13T19:23:43,088 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 1334c8616b794fd8c4fb42c4bfbc0d50, server=e4b56bc8c4b3,40371,1731525792389 in 1.6680 sec 2024-11-13T19:23:43,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-13T19:23:43,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1334c8616b794fd8c4fb42c4bfbc0d50, UNASSIGN in 1.6750 sec 2024-11-13T19:23:43,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-13T19:23:43,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6810 sec 2024-11-13T19:23:43,092 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525823092"}]},"ts":"1731525823092"} 2024-11-13T19:23:43,094 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:23:43,094 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:23:43,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6950 sec 2024-11-13T19:23:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-13T19:23:43,552 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:43,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:23:43,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:43,563 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:43,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-13T19:23:43,566 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:43,572 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:43,577 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/recovered.edits] 2024-11-13T19:23:43,580 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/44a06d9038b24f9b840655a92109b620 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/44a06d9038b24f9b840655a92109b620 2024-11-13T19:23:43,582 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b49f53ae568148fe91fee3f5427346b6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/A/b49f53ae568148fe91fee3f5427346b6 2024-11-13T19:23:43,584 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/948d34a82ed44ac8b5f6bfd55930af5c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/948d34a82ed44ac8b5f6bfd55930af5c 2024-11-13T19:23:43,585 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f41812af39d5413f95af788d6501c330 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/B/f41812af39d5413f95af788d6501c330 2024-11-13T19:23:43,588 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5e52ff24b377468e9e293a47687a8e8a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/5e52ff24b377468e9e293a47687a8e8a 2024-11-13T19:23:43,590 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f844f579a4a04189a102d47b8f0ed5c0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/C/f844f579a4a04189a102d47b8f0ed5c0 2024-11-13T19:23:43,593 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/recovered.edits/572.seqid to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50/recovered.edits/572.seqid 2024-11-13T19:23:43,593 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/1334c8616b794fd8c4fb42c4bfbc0d50 2024-11-13T19:23:43,593 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:23:43,599 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:43,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-13T19:23:43,611 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:23:43,641 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:23:43,643 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:43,643 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:23:43,644 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731525823643"}]},"ts":"9223372036854775807"} 2024-11-13T19:23:43,648 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:23:43,648 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1334c8616b794fd8c4fb42c4bfbc0d50, NAME => 'TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:23:43,648 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:23:43,649 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731525823648"}]},"ts":"9223372036854775807"} 2024-11-13T19:23:43,652 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:23:43,653 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:43,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-13T19:23:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-13T19:23:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:23:43,673 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:43,683 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=240 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349642852_22 at /127.0.0.1:60526 [Waiting for operation #316] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349642852_22 at /127.0.0.1:60016 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;e4b56bc8c4b3:40371-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-340608075_22 at /127.0.0.1:41732 [Waiting for operation #141] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-340608075_22 at /127.0.0.1:38852 [Waiting for operation #299] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=359 (was 240) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1407 (was 1275) - AvailableMemoryMB LEAK? - 2024-11-13T19:23:43,693 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=240, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=359, ProcessCount=11, AvailableMemoryMB=1536 2024-11-13T19:23:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:23:43,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:23:43,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:43,697 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:23:43,697 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:43,697 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-13T19:23:43,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:23:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:23:43,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741960_1136 (size=959) 2024-11-13T19:23:43,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:23:44,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:23:44,107 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:23:44,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741961_1137 (size=53) 2024-11-13T19:23:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:23:44,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:44,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8f3bce5c1bcfc0bd5040de3aa8d43668, disabling compactions & flushes 2024-11-13T19:23:44,513 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:44,513 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:44,514 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. after waiting 0 ms 2024-11-13T19:23:44,514 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:44,514 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:44,514 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: Waiting for close lock at 1731525824513Disabling compacts and flushes for region at 1731525824513Disabling writes for close at 1731525824514 (+1 ms)Writing region close event to WAL at 1731525824514Closed at 1731525824514 2024-11-13T19:23:44,515 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:23:44,516 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731525824515"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731525824515"}]},"ts":"1731525824515"} 2024-11-13T19:23:44,518 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:23:44,519 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:23:44,519 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525824519"}]},"ts":"1731525824519"} 2024-11-13T19:23:44,521 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:23:44,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, ASSIGN}] 2024-11-13T19:23:44,523 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, ASSIGN 2024-11-13T19:23:44,524 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, ASSIGN; state=OFFLINE, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=false 2024-11-13T19:23:44,674 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:44,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, ASSIGN because future has completed 2024-11-13T19:23:44,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:23:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:23:44,834 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:44,834 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:23:44,834 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,835 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:44,835 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,835 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,836 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,839 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3bce5c1bcfc0bd5040de3aa8d43668 columnFamilyName A 2024-11-13T19:23:44,840 DEBUG [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:44,840 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(327): Store=8f3bce5c1bcfc0bd5040de3aa8d43668/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:44,841 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,842 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3bce5c1bcfc0bd5040de3aa8d43668 columnFamilyName B 2024-11-13T19:23:44,843 DEBUG [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:44,843 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(327): Store=8f3bce5c1bcfc0bd5040de3aa8d43668/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:44,843 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,845 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3bce5c1bcfc0bd5040de3aa8d43668 columnFamilyName C 2024-11-13T19:23:44,845 DEBUG [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:44,846 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(327): Store=8f3bce5c1bcfc0bd5040de3aa8d43668/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:44,846 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,847 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,848 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,849 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,849 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,850 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:23:44,851 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,853 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:23:44,854 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 8f3bce5c1bcfc0bd5040de3aa8d43668; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74522167, jitterRate=0.11046682298183441}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:23:44,854 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:44,855 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: Running coprocessor pre-open hook at 1731525824835Writing region info on filesystem at 1731525824835Initializing all the Stores at 1731525824836 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525824836Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525824836Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525824836Cleaning up temporary data from old regions at 1731525824849 (+13 ms)Running coprocessor post-open hooks at 1731525824854 (+5 ms)Region opened successfully at 1731525824855 (+1 ms) 2024-11-13T19:23:44,857 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., pid=36, masterSystemTime=1731525824829 2024-11-13T19:23:44,863 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:44,864 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:44,864 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=OPEN, openSeqNum=2, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:44,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:23:44,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-13T19:23:44,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 in 191 msec 2024-11-13T19:23:44,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-13T19:23:44,875 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, ASSIGN in 350 msec 2024-11-13T19:23:44,876 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:23:44,876 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525824876"}]},"ts":"1731525824876"} 2024-11-13T19:23:44,878 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:23:44,880 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:23:44,882 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-13T19:23:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-13T19:23:45,842 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:45,843 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:45,843 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:45,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:45,845 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:45,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@743a41ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:45,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:45,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:45,848 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:45,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1f6f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:45,850 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:45,851 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:45,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:45,854 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:23:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:23:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7e9d1cb2 2024-11-13T19:23:45,855 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:23:45,856 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:23:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:23:45,861 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:23:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T19:23:45,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741962_1138 (size=995) 2024-11-13T19:23:46,283 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T19:23:46,283 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T19:23:46,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:23:46,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, REOPEN/MOVE}] 2024-11-13T19:23:46,299 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, REOPEN/MOVE 2024-11-13T19:23:46,300 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, REOPEN/MOVE because future has completed 2024-11-13T19:23:46,303 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:23:46,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:23:46,456 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,456 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:23:46,456 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 8f3bce5c1bcfc0bd5040de3aa8d43668, disabling compactions & flushes 2024-11-13T19:23:46,456 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:46,456 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:46,456 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. after waiting 0 ms 2024-11-13T19:23:46,456 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:46,461 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T19:23:46,461 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:46,461 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: Waiting for close lock at 1731525826456Running coprocessor pre-close hooks at 1731525826456Disabling compacts and flushes for region at 1731525826456Disabling writes for close at 1731525826456Writing region close event to WAL at 1731525826457 (+1 ms)Running coprocessor post-close hooks at 1731525826461 (+4 ms)Closed at 1731525826461 2024-11-13T19:23:46,462 WARN [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 8f3bce5c1bcfc0bd5040de3aa8d43668 to self. 2024-11-13T19:23:46,463 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=CLOSED 2024-11-13T19:23:46,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:23:46,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-13T19:23:46,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 in 164 msec 2024-11-13T19:23:46,469 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, REOPEN/MOVE; state=CLOSED, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=true 2024-11-13T19:23:46,620 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,621 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, REOPEN/MOVE because future has completed 2024-11-13T19:23:46,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:23:46,777 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:46,777 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:23:46,778 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,778 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:23:46,778 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,778 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,785 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,792 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3bce5c1bcfc0bd5040de3aa8d43668 columnFamilyName A 2024-11-13T19:23:46,794 DEBUG [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:46,794 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(327): Store=8f3bce5c1bcfc0bd5040de3aa8d43668/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:46,794 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,795 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3bce5c1bcfc0bd5040de3aa8d43668 columnFamilyName B 2024-11-13T19:23:46,795 DEBUG [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:46,796 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(327): Store=8f3bce5c1bcfc0bd5040de3aa8d43668/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:46,796 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,796 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8f3bce5c1bcfc0bd5040de3aa8d43668 columnFamilyName C 2024-11-13T19:23:46,796 DEBUG [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:46,797 INFO [StoreOpener-8f3bce5c1bcfc0bd5040de3aa8d43668-1 {}] regionserver.HStore(327): Store=8f3bce5c1bcfc0bd5040de3aa8d43668/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:23:46,797 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,798 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,799 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,800 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,800 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,800 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:23:46,802 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,802 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 8f3bce5c1bcfc0bd5040de3aa8d43668; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59555228, jitterRate=-0.11255794763565063}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:23:46,802 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,804 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: Running coprocessor pre-open hook at 1731525826779Writing region info on filesystem at 1731525826779Initializing all the Stores at 1731525826780 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525826780Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525826781 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525826781Cleaning up temporary data from old regions at 1731525826800 (+19 ms)Running coprocessor post-open hooks at 1731525826802 (+2 ms)Region opened successfully at 1731525826804 (+2 ms) 2024-11-13T19:23:46,804 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., pid=41, masterSystemTime=1731525826773 2024-11-13T19:23:46,806 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:46,807 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:46,807 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=OPEN, openSeqNum=5, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:23:46,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-13T19:23:46,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 in 188 msec 2024-11-13T19:23:46,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-13T19:23:46,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, REOPEN/MOVE in 515 msec 2024-11-13T19:23:46,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-13T19:23:46,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 529 msec 2024-11-13T19:23:46,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 956 msec 2024-11-13T19:23:46,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-13T19:23:46,825 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:46,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,826 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,826 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,827 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5557ead4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,828 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,829 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51566, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,829 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e1d6873, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,831 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e4f9348, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,833 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,834 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49f00159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,835 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,835 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,836 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,836 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b7e4f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,838 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cd59cdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,841 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12821f5e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,843 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6052db3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,845 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@242ccfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cc3083f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,849 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,849 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,849 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,850 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@782221cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,852 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b67c59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,854 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a776fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,856 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51670, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a955add, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,857 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,857 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,858 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7093a516, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,859 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a25843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,860 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,861 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:23:46,861 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:23:46,861 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:23:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:23:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:23:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b5a21b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:23:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:23:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:23:46,863 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:23:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9eb05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:23:46,865 DEBUG [Thread-688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,865 DEBUG [Thread-692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,865 DEBUG [Thread-701 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,865 DEBUG [Thread-695 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,865 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,866 DEBUG [Thread-697 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,866 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-13T19:23:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:23:46,869 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:46,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47386, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,870 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,870 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:46,870 DEBUG [Thread-684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,871 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,873 DEBUG [Thread-690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,875 DEBUG [Thread-686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,875 DEBUG [Thread-699 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:23:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:23:46,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,876 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:23:46,877 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,877 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,880 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:23:46,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:23:46,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,885 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:46,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:23:46,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111378f727f8839445e39ed70ac3e9508ba8_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525826884/Put/seqid=0 2024-11-13T19:23:46,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741963_1139 (size=12154) 2024-11-13T19:23:46,934 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:46,940 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111378f727f8839445e39ed70ac3e9508ba8_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111378f727f8839445e39ed70ac3e9508ba8_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:46,941 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/8058a37e334842c099f9b616e290834b, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:46,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/8058a37e334842c099f9b616e290834b is 175, key is test_row_0/A:col10/1731525826884/Put/seqid=0 2024-11-13T19:23:46,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:46,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525886937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,956 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:46,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525886940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525886947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:46,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525886942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:46,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:46,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525886956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:46,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:23:46,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741964_1140 (size=30955) 2024-11-13T19:23:47,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:23:47,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:47,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525887063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,064 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525887063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525887063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525887063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525887063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,066 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:23:47,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:23:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525887273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525887273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,274 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525887273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525887273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,275 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525887273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:23:47,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/8058a37e334842c099f9b616e290834b 2024-11-13T19:23:47,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/4e944e4bf41f403eb8af27734d200ff3 is 50, key is test_row_0/B:col10/1731525826884/Put/seqid=0 2024-11-13T19:23:47,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741965_1141 (size=12001) 2024-11-13T19:23:47,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:23:47,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:47,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:23:47,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525887582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525887582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525887583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525887583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:47,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525887583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:47,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:47,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:23:47,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:47,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:23:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:47,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/4e944e4bf41f403eb8af27734d200ff3 2024-11-13T19:23:47,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/f44743be15d24f42849ad47b9d2f1aa7 is 50, key is test_row_0/C:col10/1731525826884/Put/seqid=0 2024-11-13T19:23:47,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741966_1142 (size=12001) 2024-11-13T19:23:47,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/f44743be15d24f42849ad47b9d2f1aa7 2024-11-13T19:23:47,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/8058a37e334842c099f9b616e290834b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8058a37e334842c099f9b616e290834b 2024-11-13T19:23:47,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8058a37e334842c099f9b616e290834b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-13T19:23:47,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/4e944e4bf41f403eb8af27734d200ff3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/4e944e4bf41f403eb8af27734d200ff3 2024-11-13T19:23:47,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/4e944e4bf41f403eb8af27734d200ff3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:23:47,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/f44743be15d24f42849ad47b9d2f1aa7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/f44743be15d24f42849ad47b9d2f1aa7 2024-11-13T19:23:47,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/f44743be15d24f42849ad47b9d2f1aa7, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:23:47,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1021ms, sequenceid=16, compaction requested=false 2024-11-13T19:23:47,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:47,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-13T19:23:47,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:47,951 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:23:47,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ee0f0ba43bf7409ba60c70291065d327_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525826937/Put/seqid=0 2024-11-13T19:23:47,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741967_1143 (size=12154) 2024-11-13T19:23:47,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:47,999 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ee0f0ba43bf7409ba60c70291065d327_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee0f0ba43bf7409ba60c70291065d327_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:23:48,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/27af637effe24da78dc3079fabaa8a54, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/27af637effe24da78dc3079fabaa8a54 is 175, key is test_row_0/A:col10/1731525826937/Put/seqid=0 2024-11-13T19:23:48,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741968_1144 (size=30955) 2024-11-13T19:23:48,020 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/27af637effe24da78dc3079fabaa8a54 2024-11-13T19:23:48,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/154d3b5823234cb996114ef9f5b3a022 is 50, key is test_row_0/B:col10/1731525826937/Put/seqid=0 2024-11-13T19:23:48,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741969_1145 (size=12001) 2024-11-13T19:23:48,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:48,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525888105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525888107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525888110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525888113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,116 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525888113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525888222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525888222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525888223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525888223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525888223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,225 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525888431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525888431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525888431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525888432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525888432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,434 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,457 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/154d3b5823234cb996114ef9f5b3a022 2024-11-13T19:23:48,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/14c0b6add5fb48dcaaa802201f9efed7 is 50, key is test_row_0/C:col10/1731525826937/Put/seqid=0 2024-11-13T19:23:48,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741970_1146 (size=12001) 2024-11-13T19:23:48,634 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:23:48,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525888741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525888741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525888741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,744 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525888742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:48,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525888742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:48,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:48,893 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/14c0b6add5fb48dcaaa802201f9efed7 2024-11-13T19:23:48,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/27af637effe24da78dc3079fabaa8a54 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/27af637effe24da78dc3079fabaa8a54 2024-11-13T19:23:48,911 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/27af637effe24da78dc3079fabaa8a54, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T19:23:48,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/154d3b5823234cb996114ef9f5b3a022 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/154d3b5823234cb996114ef9f5b3a022 2024-11-13T19:23:48,919 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/154d3b5823234cb996114ef9f5b3a022, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T19:23:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/14c0b6add5fb48dcaaa802201f9efed7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/14c0b6add5fb48dcaaa802201f9efed7 2024-11-13T19:23:48,926 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/14c0b6add5fb48dcaaa802201f9efed7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T19:23:48,927 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 976ms, sequenceid=40, compaction requested=false 2024-11-13T19:23:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:48,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-13T19:23:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-13T19:23:48,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-13T19:23:48,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0590 sec 2024-11-13T19:23:48,934 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.0670 sec 2024-11-13T19:23:49,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-13T19:23:49,011 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:49,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-13T19:23:49,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:23:49,014 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:49,015 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:49,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:49,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:23:49,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-13T19:23:49,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:49,169 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T19:23:49,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a44faba4cf114029b26f3746570a93be_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525828110/Put/seqid=0 2024-11-13T19:23:49,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741971_1147 (size=12154) 2024-11-13T19:23:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:49,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:49,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525889292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525889292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525889294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525889294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525889295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:23:49,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525889401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525889401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525889412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525889413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525889413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:49,589 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a44faba4cf114029b26f3746570a93be_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a44faba4cf114029b26f3746570a93be_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/f2410070e26a4bb38dfdfc165359c7ff, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:49,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/f2410070e26a4bb38dfdfc165359c7ff is 175, key is test_row_0/A:col10/1731525828110/Put/seqid=0 2024-11-13T19:23:49,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741972_1148 (size=30955) 2024-11-13T19:23:49,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525889612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525889612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525889621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525889621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525889621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,625 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:23:49,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525889922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525889922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525889931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,936 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525889931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525889932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:50,004 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/f2410070e26a4bb38dfdfc165359c7ff 2024-11-13T19:23:50,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/cb5baed9dfbc4a37834cb2f342ca73bd is 50, key is test_row_0/B:col10/1731525828110/Put/seqid=0 2024-11-13T19:23:50,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741973_1149 (size=12001) 2024-11-13T19:23:50,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:23:50,419 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/cb5baed9dfbc4a37834cb2f342ca73bd 2024-11-13T19:23:50,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/97048f9d704746c1b7b10b54f65c9bec is 50, key is test_row_0/C:col10/1731525828110/Put/seqid=0 2024-11-13T19:23:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741974_1150 (size=12001) 2024-11-13T19:23:50,436 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/97048f9d704746c1b7b10b54f65c9bec 2024-11-13T19:23:50,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:50,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525890442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525890442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525890442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:50,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525890442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:50,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/f2410070e26a4bb38dfdfc165359c7ff as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/f2410070e26a4bb38dfdfc165359c7ff 2024-11-13T19:23:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:50,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525890451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:50,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:50,457 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/f2410070e26a4bb38dfdfc165359c7ff, entries=150, sequenceid=52, filesize=30.2 K 2024-11-13T19:23:50,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/cb5baed9dfbc4a37834cb2f342ca73bd as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/cb5baed9dfbc4a37834cb2f342ca73bd 2024-11-13T19:23:50,463 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/cb5baed9dfbc4a37834cb2f342ca73bd, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T19:23:50,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/97048f9d704746c1b7b10b54f65c9bec as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/97048f9d704746c1b7b10b54f65c9bec 2024-11-13T19:23:50,471 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/97048f9d704746c1b7b10b54f65c9bec, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T19:23:50,473 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1304ms, sequenceid=52, compaction requested=true 2024-11-13T19:23:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:50,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-13T19:23:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-13T19:23:50,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-13T19:23:50,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-13T19:23:50,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.4650 sec 2024-11-13T19:23:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-13T19:23:51,162 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:51,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-13T19:23:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T19:23:51,164 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:51,165 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:51,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T19:23:51,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-13T19:23:51,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:51,318 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:23:51,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cba611fc9cab4264a190142ad768747a_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525829290/Put/seqid=0 2024-11-13T19:23:51,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741975_1151 (size=12154) 2024-11-13T19:23:51,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:51,337 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113cba611fc9cab4264a190142ad768747a_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cba611fc9cab4264a190142ad768747a_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:51,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/cef17d35eb434aeb989a0d569db0e4d3, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:51,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/cef17d35eb434aeb989a0d569db0e4d3 is 175, key is test_row_0/A:col10/1731525829290/Put/seqid=0 2024-11-13T19:23:51,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741976_1152 (size=30955) 2024-11-13T19:23:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:51,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:51,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525891464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525891464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525891466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,472 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525891466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525891467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,473 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T19:23:51,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525891582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525891582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525891582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525891582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525891582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,586 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,744 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/cef17d35eb434aeb989a0d569db0e4d3 2024-11-13T19:23:51,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/de4473c19db8488a9fd8513a5dfbf742 is 50, key is test_row_0/B:col10/1731525829290/Put/seqid=0 2024-11-13T19:23:51,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741977_1153 (size=12001) 2024-11-13T19:23:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T19:23:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525891791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525891791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525891791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525891791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525891791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,794 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:51,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525892101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525892101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525892101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525892101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,103 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525892101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,157 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/de4473c19db8488a9fd8513a5dfbf742 2024-11-13T19:23:52,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/4f1d8cc83ba74148b963951cc3f81e34 is 50, key is test_row_0/C:col10/1731525829290/Put/seqid=0 2024-11-13T19:23:52,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741978_1154 (size=12001) 2024-11-13T19:23:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T19:23:52,574 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/4f1d8cc83ba74148b963951cc3f81e34 2024-11-13T19:23:52,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/cef17d35eb434aeb989a0d569db0e4d3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/cef17d35eb434aeb989a0d569db0e4d3 2024-11-13T19:23:52,584 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/cef17d35eb434aeb989a0d569db0e4d3, entries=150, sequenceid=76, filesize=30.2 K 2024-11-13T19:23:52,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/de4473c19db8488a9fd8513a5dfbf742 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/de4473c19db8488a9fd8513a5dfbf742 2024-11-13T19:23:52,590 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/de4473c19db8488a9fd8513a5dfbf742, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T19:23:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/4f1d8cc83ba74148b963951cc3f81e34 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4f1d8cc83ba74148b963951cc3f81e34 2024-11-13T19:23:52,597 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4f1d8cc83ba74148b963951cc3f81e34, entries=150, sequenceid=76, filesize=11.7 K 2024-11-13T19:23:52,598 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1280ms, sequenceid=76, compaction requested=true 2024-11-13T19:23:52,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:52,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:52,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-13T19:23:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-13T19:23:52,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-13T19:23:52,603 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4360 sec 2024-11-13T19:23:52,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4420 sec 2024-11-13T19:23:52,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:52,615 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:52,615 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T19:23:52,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134e7a7600ade144769181c6c19ee90efb_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525831457/Put/seqid=0 2024-11-13T19:23:52,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741979_1155 (size=14594) 2024-11-13T19:23:52,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525892674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525892674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525892678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525892679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525892681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525892782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525892791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525892791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525892791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525892801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:52,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:52,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525892991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:52,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525893001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525893002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525893002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,005 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525893011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:53,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411134e7a7600ade144769181c6c19ee90efb_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134e7a7600ade144769181c6c19ee90efb_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:53,043 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/59dc5c6762954880ae37fed346099528, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:53,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/59dc5c6762954880ae37fed346099528 is 175, key is test_row_0/A:col10/1731525831457/Put/seqid=0 2024-11-13T19:23:53,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741980_1156 (size=39549) 2024-11-13T19:23:53,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525893301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,305 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-13T19:23:53,312 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:53,312 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-13T19:23:53,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:23:53,314 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:53,314 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:53,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:53,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525893312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525893312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525893312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525893321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:23:53,447 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/59dc5c6762954880ae37fed346099528 2024-11-13T19:23:53,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/f2e2712a6c3449ef88427e572afac53b is 50, key is test_row_0/B:col10/1731525831457/Put/seqid=0 2024-11-13T19:23:53,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741981_1157 (size=12001) 2024-11-13T19:23:53,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T19:23:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T19:23:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:23:53,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T19:23:53,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525893811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525893821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525893821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525893831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,837 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525893832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:53,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:53,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/f2e2712a6c3449ef88427e572afac53b 2024-11-13T19:23:53,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/9de7e9aaeff841d7bf044f8216859149 is 50, key is test_row_0/C:col10/1731525831457/Put/seqid=0 2024-11-13T19:23:53,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741982_1158 (size=12001) 2024-11-13T19:23:53,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T19:23:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:53,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:53,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:53,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:23:54,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T19:23:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:54,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:54,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:54,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:54,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T19:23:54,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:54,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:54,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:54,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/9de7e9aaeff841d7bf044f8216859149 2024-11-13T19:23:54,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/59dc5c6762954880ae37fed346099528 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/59dc5c6762954880ae37fed346099528 2024-11-13T19:23:54,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/59dc5c6762954880ae37fed346099528, entries=200, sequenceid=89, filesize=38.6 K 2024-11-13T19:23:54,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/f2e2712a6c3449ef88427e572afac53b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/f2e2712a6c3449ef88427e572afac53b 2024-11-13T19:23:54,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/f2e2712a6c3449ef88427e572afac53b, entries=150, sequenceid=89, filesize=11.7 K 2024-11-13T19:23:54,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/9de7e9aaeff841d7bf044f8216859149 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9de7e9aaeff841d7bf044f8216859149 2024-11-13T19:23:54,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9de7e9aaeff841d7bf044f8216859149, entries=150, sequenceid=89, filesize=11.7 K 2024-11-13T19:23:54,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1677ms, sequenceid=89, compaction requested=true 2024-11-13T19:23:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:54,292 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:23:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:54,292 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:23:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:54,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:54,294 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:23:54,294 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/B is initiating minor compaction (all files) 2024-11-13T19:23:54,294 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/B in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,294 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/4e944e4bf41f403eb8af27734d200ff3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/154d3b5823234cb996114ef9f5b3a022, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/cb5baed9dfbc4a37834cb2f342ca73bd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/de4473c19db8488a9fd8513a5dfbf742, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/f2e2712a6c3449ef88427e572afac53b] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=58.6 K 2024-11-13T19:23:54,294 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 163369 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:23:54,294 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/A is initiating minor compaction (all files) 2024-11-13T19:23:54,294 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/A in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,294 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8058a37e334842c099f9b616e290834b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/27af637effe24da78dc3079fabaa8a54, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/f2410070e26a4bb38dfdfc165359c7ff, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/cef17d35eb434aeb989a0d569db0e4d3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/59dc5c6762954880ae37fed346099528] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=159.5 K 2024-11-13T19:23:54,295 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,295 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8058a37e334842c099f9b616e290834b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/27af637effe24da78dc3079fabaa8a54, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/f2410070e26a4bb38dfdfc165359c7ff, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/cef17d35eb434aeb989a0d569db0e4d3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/59dc5c6762954880ae37fed346099528] 2024-11-13T19:23:54,295 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e944e4bf41f403eb8af27734d200ff3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731525826883 2024-11-13T19:23:54,295 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8058a37e334842c099f9b616e290834b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731525826883 2024-11-13T19:23:54,296 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 154d3b5823234cb996114ef9f5b3a022, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525826937 2024-11-13T19:23:54,296 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27af637effe24da78dc3079fabaa8a54, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525826937 2024-11-13T19:23:54,296 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting cb5baed9dfbc4a37834cb2f342ca73bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525828104 2024-11-13T19:23:54,296 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2410070e26a4bb38dfdfc165359c7ff, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525828104 2024-11-13T19:23:54,296 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting de4473c19db8488a9fd8513a5dfbf742, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731525829290 2024-11-13T19:23:54,296 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef17d35eb434aeb989a0d569db0e4d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731525829290 2024-11-13T19:23:54,296 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e2712a6c3449ef88427e572afac53b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731525831457 2024-11-13T19:23:54,297 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59dc5c6762954880ae37fed346099528, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731525831457 2024-11-13T19:23:54,307 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:54,308 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#B#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:54,308 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/61551e3c396b4f9d824399c524fdb2ab is 50, key is test_row_0/B:col10/1731525831457/Put/seqid=0 2024-11-13T19:23:54,311 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113bb1fe77f0ec849ac93237c149fb4222d_8f3bce5c1bcfc0bd5040de3aa8d43668 store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:54,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741983_1159 (size=12173) 2024-11-13T19:23:54,316 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113bb1fe77f0ec849ac93237c149fb4222d_8f3bce5c1bcfc0bd5040de3aa8d43668, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:54,317 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb1fe77f0ec849ac93237c149fb4222d_8f3bce5c1bcfc0bd5040de3aa8d43668 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:54,322 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/61551e3c396b4f9d824399c524fdb2ab as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/61551e3c396b4f9d824399c524fdb2ab 2024-11-13T19:23:54,329 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/B of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 61551e3c396b4f9d824399c524fdb2ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:54,329 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:54,329 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/B, priority=11, startTime=1731525834292; duration=0sec 2024-11-13T19:23:54,329 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:54,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741984_1160 (size=4469) 2024-11-13T19:23:54,329 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:B 2024-11-13T19:23:54,329 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-13T19:23:54,334 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#A#compaction#139 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:54,334 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/9cdec6f41dee4b8b800751bbd6dea0b9 is 175, key is test_row_0/A:col10/1731525831457/Put/seqid=0 2024-11-13T19:23:54,335 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60005 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-13T19:23:54,335 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/C is initiating minor compaction (all files) 2024-11-13T19:23:54,335 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/C in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,335 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/f44743be15d24f42849ad47b9d2f1aa7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/14c0b6add5fb48dcaaa802201f9efed7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/97048f9d704746c1b7b10b54f65c9bec, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4f1d8cc83ba74148b963951cc3f81e34, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9de7e9aaeff841d7bf044f8216859149] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=58.6 K 2024-11-13T19:23:54,336 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f44743be15d24f42849ad47b9d2f1aa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731525826883 2024-11-13T19:23:54,336 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 14c0b6add5fb48dcaaa802201f9efed7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525826937 2024-11-13T19:23:54,337 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 97048f9d704746c1b7b10b54f65c9bec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525828104 2024-11-13T19:23:54,337 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f1d8cc83ba74148b963951cc3f81e34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731525829290 2024-11-13T19:23:54,338 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9de7e9aaeff841d7bf044f8216859149, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731525831457 2024-11-13T19:23:54,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741985_1161 (size=31127) 2024-11-13T19:23:54,352 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#C#compaction#140 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:54,352 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/9601188f9b4849449675cef67ff26710 is 50, key is test_row_0/C:col10/1731525831457/Put/seqid=0 2024-11-13T19:23:54,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741986_1162 (size=12173) 2024-11-13T19:23:54,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-13T19:23:54,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:54,385 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:23:54,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131bce309084ba45df939f65eb5a48689b_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525832677/Put/seqid=0 2024-11-13T19:23:54,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741987_1163 (size=12154) 2024-11-13T19:23:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:23:54,755 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/9cdec6f41dee4b8b800751bbd6dea0b9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9cdec6f41dee4b8b800751bbd6dea0b9 2024-11-13T19:23:54,767 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/9601188f9b4849449675cef67ff26710 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9601188f9b4849449675cef67ff26710 2024-11-13T19:23:54,770 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/A of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 9cdec6f41dee4b8b800751bbd6dea0b9(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:54,771 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:54,774 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/A, priority=11, startTime=1731525834292; duration=0sec 2024-11-13T19:23:54,776 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/C of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 9601188f9b4849449675cef67ff26710(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:54,776 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:54,776 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/C, priority=11, startTime=1731525834292; duration=0sec 2024-11-13T19:23:54,776 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:54,776 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:C 2024-11-13T19:23:54,777 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:54,777 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:A 2024-11-13T19:23:54,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:54,804 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131bce309084ba45df939f65eb5a48689b_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131bce309084ba45df939f65eb5a48689b_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:54,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/c37bc0e54f21408487b3843eb819757b, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:54,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/c37bc0e54f21408487b3843eb819757b is 175, key is test_row_0/A:col10/1731525832677/Put/seqid=0 2024-11-13T19:23:54,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741988_1164 (size=30955) 2024-11-13T19:23:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:54,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:54,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525894847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525894848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525894848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525894849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525894852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,857 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 8f3bce5c1bcfc0bd5040de3aa8d43668 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525894961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525894961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525894962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525894962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:54,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525895171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525895171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525895172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525895172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,229 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/c37bc0e54f21408487b3843eb819757b 2024-11-13T19:23:55,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/83e171fb36ef4e21bf83668ba27d0584 is 50, key is test_row_0/B:col10/1731525832677/Put/seqid=0 2024-11-13T19:23:55,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741989_1165 (size=12001) 2024-11-13T19:23:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:23:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525895481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525895481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525895482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525895482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,485 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:55,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/83e171fb36ef4e21bf83668ba27d0584 2024-11-13T19:23:55,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/caf3e000ac01462f8e416917a9260db9 is 50, key is test_row_0/C:col10/1731525832677/Put/seqid=0 2024-11-13T19:23:55,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741990_1166 (size=12001) 2024-11-13T19:23:55,659 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/caf3e000ac01462f8e416917a9260db9 2024-11-13T19:23:55,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/c37bc0e54f21408487b3843eb819757b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/c37bc0e54f21408487b3843eb819757b 2024-11-13T19:23:55,670 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/c37bc0e54f21408487b3843eb819757b, entries=150, sequenceid=114, filesize=30.2 K 2024-11-13T19:23:55,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/83e171fb36ef4e21bf83668ba27d0584 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/83e171fb36ef4e21bf83668ba27d0584 2024-11-13T19:23:55,678 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/83e171fb36ef4e21bf83668ba27d0584, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:23:55,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/caf3e000ac01462f8e416917a9260db9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/caf3e000ac01462f8e416917a9260db9 2024-11-13T19:23:55,684 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/caf3e000ac01462f8e416917a9260db9, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:23:55,686 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1302ms, sequenceid=114, compaction requested=false 2024-11-13T19:23:55,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:55,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:55,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-13T19:23:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-13T19:23:55,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-13T19:23:55,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3740 sec 2024-11-13T19:23:55,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3780 sec 2024-11-13T19:23:55,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:55,994 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:55,994 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T19:23:56,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111380752e1546f6447ca185b5eae539456a_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525834846/Put/seqid=0 2024-11-13T19:23:56,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741991_1167 (size=12204) 2024-11-13T19:23:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525896019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525896020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525896023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525896025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525896131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525896131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525896131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525896131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525896341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525896341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525896341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525896341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,408 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:56,480 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111380752e1546f6447ca185b5eae539456a_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111380752e1546f6447ca185b5eae539456a_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:56,481 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0f2b7d8fb8d64963b02f2fb65a08a8db, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:56,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0f2b7d8fb8d64963b02f2fb65a08a8db is 175, key is test_row_0/A:col10/1731525834846/Put/seqid=0 2024-11-13T19:23:56,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741992_1168 (size=31005) 2024-11-13T19:23:56,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525896651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525896651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525896651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525896651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525896881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:56,883 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 8f3bce5c1bcfc0bd5040de3aa8d43668 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:56,885 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=131, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0f2b7d8fb8d64963b02f2fb65a08a8db 2024-11-13T19:23:56,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/8764c02ff4564d4ea25909eb55a78c16 is 50, key is test_row_0/B:col10/1731525834846/Put/seqid=0 2024-11-13T19:23:56,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741993_1169 (size=12051) 2024-11-13T19:23:57,147 INFO [master/e4b56bc8c4b3:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T19:23:57,147 INFO [master/e4b56bc8c4b3:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T19:23:57,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525897161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:57,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525897162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525897162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:57,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525897162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:57,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:57,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/8764c02ff4564d4ea25909eb55a78c16 2024-11-13T19:23:57,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/03da0c6c8edf4ddea07db56b3bd92f45 is 50, key is test_row_0/C:col10/1731525834846/Put/seqid=0 2024-11-13T19:23:57,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741994_1170 (size=12051) 2024-11-13T19:23:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-13T19:23:57,482 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:57,482 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-13T19:23:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T19:23:57,487 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:57,487 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:57,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:57,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T19:23:57,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T19:23:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:57,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:57,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:57,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:57,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/03da0c6c8edf4ddea07db56b3bd92f45 2024-11-13T19:23:57,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0f2b7d8fb8d64963b02f2fb65a08a8db as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0f2b7d8fb8d64963b02f2fb65a08a8db 2024-11-13T19:23:57,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0f2b7d8fb8d64963b02f2fb65a08a8db, entries=150, sequenceid=131, filesize=30.3 K 2024-11-13T19:23:57,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/8764c02ff4564d4ea25909eb55a78c16 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/8764c02ff4564d4ea25909eb55a78c16 2024-11-13T19:23:57,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/8764c02ff4564d4ea25909eb55a78c16, entries=150, sequenceid=131, filesize=11.8 K 2024-11-13T19:23:57,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/03da0c6c8edf4ddea07db56b3bd92f45 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/03da0c6c8edf4ddea07db56b3bd92f45 2024-11-13T19:23:57,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/03da0c6c8edf4ddea07db56b3bd92f45, entries=150, sequenceid=131, filesize=11.8 K 2024-11-13T19:23:57,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1756ms, sequenceid=131, compaction requested=true 2024-11-13T19:23:57,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:57,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:57,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:57,750 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:57,750 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:57,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:57,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:57,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:57,751 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:57,753 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:57,753 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93087 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:57,753 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/B is initiating minor compaction (all files) 2024-11-13T19:23:57,753 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/A is initiating minor compaction (all files) 2024-11-13T19:23:57,753 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/A in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:57,753 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9cdec6f41dee4b8b800751bbd6dea0b9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/c37bc0e54f21408487b3843eb819757b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0f2b7d8fb8d64963b02f2fb65a08a8db] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=90.9 K 2024-11-13T19:23:57,753 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:57,754 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9cdec6f41dee4b8b800751bbd6dea0b9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/c37bc0e54f21408487b3843eb819757b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0f2b7d8fb8d64963b02f2fb65a08a8db] 2024-11-13T19:23:57,754 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/B in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:57,754 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/61551e3c396b4f9d824399c524fdb2ab, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/83e171fb36ef4e21bf83668ba27d0584, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/8764c02ff4564d4ea25909eb55a78c16] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=35.4 K 2024-11-13T19:23:57,754 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cdec6f41dee4b8b800751bbd6dea0b9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731525831457 2024-11-13T19:23:57,754 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 61551e3c396b4f9d824399c524fdb2ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731525831457 2024-11-13T19:23:57,755 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 83e171fb36ef4e21bf83668ba27d0584, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525832671 2024-11-13T19:23:57,755 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8764c02ff4564d4ea25909eb55a78c16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731525834846 2024-11-13T19:23:57,755 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c37bc0e54f21408487b3843eb819757b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525832671 2024-11-13T19:23:57,756 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f2b7d8fb8d64963b02f2fb65a08a8db, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731525834846 2024-11-13T19:23:57,766 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:57,766 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/88c6ef2e7a2944dea80020e29d8b3b11 is 50, key is test_row_0/B:col10/1731525834846/Put/seqid=0 2024-11-13T19:23:57,779 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:57,786 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113d94e3225eaf24f40b301519c48bf7b2d_8f3bce5c1bcfc0bd5040de3aa8d43668 store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:57,790 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113d94e3225eaf24f40b301519c48bf7b2d_8f3bce5c1bcfc0bd5040de3aa8d43668, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:57,790 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113d94e3225eaf24f40b301519c48bf7b2d_8f3bce5c1bcfc0bd5040de3aa8d43668 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:57,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741995_1171 (size=12325) 2024-11-13T19:23:57,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-13T19:23:57,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:57,794 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T19:23:57,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T19:23:57,829 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/88c6ef2e7a2944dea80020e29d8b3b11 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/88c6ef2e7a2944dea80020e29d8b3b11 2024-11-13T19:23:57,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130ae887df912d4727b6658dcee063b5bb_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525836007/Put/seqid=0 2024-11-13T19:23:57,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741996_1172 (size=4469) 2024-11-13T19:23:57,838 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#A#compaction#148 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:57,838 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/51438133451e4fe19da6b6e0034fe8ad is 175, key is test_row_0/A:col10/1731525834846/Put/seqid=0 2024-11-13T19:23:57,848 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/B of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 88c6ef2e7a2944dea80020e29d8b3b11(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:57,849 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:57,849 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/B, priority=13, startTime=1731525837750; duration=0sec 2024-11-13T19:23:57,849 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:57,849 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:B 2024-11-13T19:23:57,849 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:57,851 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36225 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:57,852 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/C is initiating minor compaction (all files) 2024-11-13T19:23:57,852 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/C in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:57,852 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9601188f9b4849449675cef67ff26710, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/caf3e000ac01462f8e416917a9260db9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/03da0c6c8edf4ddea07db56b3bd92f45] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=35.4 K 2024-11-13T19:23:57,853 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9601188f9b4849449675cef67ff26710, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731525831457 2024-11-13T19:23:57,854 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting caf3e000ac01462f8e416917a9260db9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525832671 2024-11-13T19:23:57,860 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 03da0c6c8edf4ddea07db56b3bd92f45, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731525834846 2024-11-13T19:23:57,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741997_1173 (size=12304) 2024-11-13T19:23:57,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:57,881 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411130ae887df912d4727b6658dcee063b5bb_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ae887df912d4727b6658dcee063b5bb_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/b36c75bde3154299a34c0cbec786cf25, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:57,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/b36c75bde3154299a34c0cbec786cf25 is 175, key is test_row_0/A:col10/1731525836007/Put/seqid=0 2024-11-13T19:23:57,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741998_1174 (size=31279) 2024-11-13T19:23:57,893 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#C#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:57,893 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/2359a7aa59a4479b8fa709c4a108a4df is 50, key is test_row_0/C:col10/1731525834846/Put/seqid=0 2024-11-13T19:23:57,900 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/51438133451e4fe19da6b6e0034fe8ad as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/51438133451e4fe19da6b6e0034fe8ad 2024-11-13T19:23:57,908 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/A of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 51438133451e4fe19da6b6e0034fe8ad(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:57,908 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:57,908 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/A, priority=13, startTime=1731525837750; duration=0sec 2024-11-13T19:23:57,908 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:57,908 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:A 2024-11-13T19:23:57,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741999_1175 (size=31105) 2024-11-13T19:23:57,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742000_1176 (size=12325) 2024-11-13T19:23:57,980 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/2359a7aa59a4479b8fa709c4a108a4df as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/2359a7aa59a4479b8fa709c4a108a4df 2024-11-13T19:23:57,988 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/C of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 2359a7aa59a4479b8fa709c4a108a4df(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:23:57,988 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:57,988 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/C, priority=13, startTime=1731525837750; duration=0sec 2024-11-13T19:23:57,988 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:57,988 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:C 2024-11-13T19:23:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T19:23:58,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:58,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525898199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525898200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525898200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525898201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,206 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525898312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525898312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525898312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525898315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,343 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/b36c75bde3154299a34c0cbec786cf25 2024-11-13T19:23:58,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/2639756fcc1f43ccac2f8263d63202f3 is 50, key is test_row_0/B:col10/1731525836007/Put/seqid=0 2024-11-13T19:23:58,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742001_1177 (size=12151) 2024-11-13T19:23:58,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525898522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525898523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525898526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,527 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525898527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T19:23:58,775 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/2639756fcc1f43ccac2f8263d63202f3 2024-11-13T19:23:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/367a0faa656345c9ab46eb49a108cefe is 50, key is test_row_0/C:col10/1731525836007/Put/seqid=0 2024-11-13T19:23:58,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525898834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525898834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525898834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525898835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:58,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:58,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742002_1178 (size=12151) 2024-11-13T19:23:59,272 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/367a0faa656345c9ab46eb49a108cefe 2024-11-13T19:23:59,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/b36c75bde3154299a34c0cbec786cf25 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/b36c75bde3154299a34c0cbec786cf25 2024-11-13T19:23:59,292 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/b36c75bde3154299a34c0cbec786cf25, entries=150, sequenceid=152, filesize=30.4 K 2024-11-13T19:23:59,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/2639756fcc1f43ccac2f8263d63202f3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/2639756fcc1f43ccac2f8263d63202f3 2024-11-13T19:23:59,305 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/2639756fcc1f43ccac2f8263d63202f3, entries=150, sequenceid=152, filesize=11.9 K 2024-11-13T19:23:59,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/367a0faa656345c9ab46eb49a108cefe as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/367a0faa656345c9ab46eb49a108cefe 2024-11-13T19:23:59,316 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/367a0faa656345c9ab46eb49a108cefe, entries=150, sequenceid=152, filesize=11.9 K 2024-11-13T19:23:59,319 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1525ms, sequenceid=152, compaction requested=false 2024-11-13T19:23:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-13T19:23:59,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-13T19:23:59,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-13T19:23:59,323 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-13T19:23:59,325 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8420 sec 2024-11-13T19:23:59,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:59,349 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:59,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T19:23:59,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371a4bd5e16f5463bbaa149be86350809_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525838195/Put/seqid=0 2024-11-13T19:23:59,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742003_1179 (size=14794) 2024-11-13T19:23:59,404 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:23:59,413 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371a4bd5e16f5463bbaa149be86350809_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371a4bd5e16f5463bbaa149be86350809_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:59,423 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/05f565096e3b457f9ccb467e5d31807d, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:59,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/05f565096e3b457f9ccb467e5d31807d is 175, key is test_row_0/A:col10/1731525838195/Put/seqid=0 2024-11-13T19:23:59,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525899411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525899411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525899423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742004_1180 (size=39749) 2024-11-13T19:23:59,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525899427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,441 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/05f565096e3b457f9ccb467e5d31807d 2024-11-13T19:23:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,442 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e00b4b3c701c41588ccd73d0f248d09d is 50, key is test_row_0/B:col10/1731525838195/Put/seqid=0 2024-11-13T19:23:59,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742005_1181 (size=12151) 2024-11-13T19:23:59,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e00b4b3c701c41588ccd73d0f248d09d 2024-11-13T19:23:59,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/61b72302450346fda32e9327789ebb99 is 50, key is test_row_0/C:col10/1731525838195/Put/seqid=0 2024-11-13T19:23:59,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525899532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742006_1182 (size=12151) 2024-11-13T19:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/61b72302450346fda32e9327789ebb99 2024-11-13T19:23:59,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525899542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525899547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525899551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/05f565096e3b457f9ccb467e5d31807d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/05f565096e3b457f9ccb467e5d31807d 2024-11-13T19:23:59,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/05f565096e3b457f9ccb467e5d31807d, entries=200, sequenceid=171, filesize=38.8 K 2024-11-13T19:23:59,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e00b4b3c701c41588ccd73d0f248d09d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e00b4b3c701c41588ccd73d0f248d09d 2024-11-13T19:23:59,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e00b4b3c701c41588ccd73d0f248d09d, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T19:23:59,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/61b72302450346fda32e9327789ebb99 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/61b72302450346fda32e9327789ebb99 2024-11-13T19:23:59,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/61b72302450346fda32e9327789ebb99, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T19:23:59,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 236ms, sequenceid=171, compaction requested=true 2024-11-13T19:23:59,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:23:59,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:23:59,585 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:23:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:23:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:23:59,586 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:23:59,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:23:59,588 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:59,588 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/B is initiating minor compaction (all files) 2024-11-13T19:23:59,588 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/B in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,588 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/88c6ef2e7a2944dea80020e29d8b3b11, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/2639756fcc1f43ccac2f8263d63202f3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e00b4b3c701c41588ccd73d0f248d09d] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=35.8 K 2024-11-13T19:23:59,589 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 88c6ef2e7a2944dea80020e29d8b3b11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731525834846 2024-11-13T19:23:59,591 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102133 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:23:59,591 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/A is initiating minor compaction (all files) 2024-11-13T19:23:59,591 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/A in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,592 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/51438133451e4fe19da6b6e0034fe8ad, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/b36c75bde3154299a34c0cbec786cf25, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/05f565096e3b457f9ccb467e5d31807d] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=99.7 K 2024-11-13T19:23:59,595 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2639756fcc1f43ccac2f8263d63202f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731525836007 2024-11-13T19:23:59,592 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,595 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/51438133451e4fe19da6b6e0034fe8ad, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/b36c75bde3154299a34c0cbec786cf25, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/05f565096e3b457f9ccb467e5d31807d] 2024-11-13T19:23:59,596 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e00b4b3c701c41588ccd73d0f248d09d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525838195 2024-11-13T19:23:59,596 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51438133451e4fe19da6b6e0034fe8ad, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731525834846 2024-11-13T19:23:59,597 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b36c75bde3154299a34c0cbec786cf25, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731525836007 2024-11-13T19:23:59,597 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05f565096e3b457f9ccb467e5d31807d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525838195 2024-11-13T19:23:59,620 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:23:59,620 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/d01cef1912bf4853ad0f766f0b6e2bb5 is 50, key is test_row_0/B:col10/1731525838195/Put/seqid=0 2024-11-13T19:23:59,623 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:59,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-13T19:23:59,632 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:23:59,633 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:23:59,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-13T19:23:59,641 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:23:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:23:59,643 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:23:59,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:23:59,651 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411133b7a37fd15ff4003aabef7234bc41136_8f3bce5c1bcfc0bd5040de3aa8d43668 store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:59,654 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411133b7a37fd15ff4003aabef7234bc41136_8f3bce5c1bcfc0bd5040de3aa8d43668, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:59,654 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133b7a37fd15ff4003aabef7234bc41136_8f3bce5c1bcfc0bd5040de3aa8d43668 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:23:59,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742007_1183 (size=12527) 2024-11-13T19:23:59,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742008_1184 (size=4469) 2024-11-13T19:23:59,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:23:59,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:23:59,758 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:23:59,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T19:23:59,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133c2ac89d903d43a598397b16b02b800e_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525839413/Put/seqid=0 2024-11-13T19:23:59,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:23:59,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:59,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:59,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:59,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525899790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525899800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525899801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525899804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742009_1185 (size=14794) 2024-11-13T19:23:59,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525899911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525899921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525899921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:23:59,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525899921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:23:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,930 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:23:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:23:59,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:23:59,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:23:59,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:23:59,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:59,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:59,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:23:59,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:24:00,098 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/d01cef1912bf4853ad0f766f0b6e2bb5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d01cef1912bf4853ad0f766f0b6e2bb5 2024-11-13T19:24:00,107 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/B of 8f3bce5c1bcfc0bd5040de3aa8d43668 into d01cef1912bf4853ad0f766f0b6e2bb5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:00,107 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:00,107 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/B, priority=13, startTime=1731525839586; duration=0sec 2024-11-13T19:24:00,107 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:00,107 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:B 2024-11-13T19:24:00,108 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:00,109 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#A#compaction#157 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:00,110 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/4dc4ab21dd114af0bd1be4f544ec343a is 175, key is test_row_0/A:col10/1731525838195/Put/seqid=0 2024-11-13T19:24:00,110 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,113 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:00,113 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/C is initiating minor compaction (all files) 2024-11-13T19:24:00,113 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/C in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,114 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/2359a7aa59a4479b8fa709c4a108a4df, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/367a0faa656345c9ab46eb49a108cefe, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/61b72302450346fda32e9327789ebb99] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=35.8 K 2024-11-13T19:24:00,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,114 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2359a7aa59a4479b8fa709c4a108a4df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731525834846 2024-11-13T19:24:00,115 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 367a0faa656345c9ab46eb49a108cefe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731525836007 2024-11-13T19:24:00,115 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b72302450346fda32e9327789ebb99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525838195 2024-11-13T19:24:00,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525900121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525900142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,146 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#C#compaction#159 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525900142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,146 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/871c4215992b4acabb86bcfef4867879 is 50, key is test_row_0/C:col10/1731525838195/Put/seqid=0 2024-11-13T19:24:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525900143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742010_1186 (size=31481) 2024-11-13T19:24:00,172 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/4dc4ab21dd114af0bd1be4f544ec343a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/4dc4ab21dd114af0bd1be4f544ec343a 2024-11-13T19:24:00,186 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/A of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 4dc4ab21dd114af0bd1be4f544ec343a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:00,186 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:00,186 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/A, priority=13, startTime=1731525839585; duration=0sec 2024-11-13T19:24:00,186 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:00,186 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:A 2024-11-13T19:24:00,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742011_1187 (size=12527) 2024-11-13T19:24:00,218 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:00,243 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133c2ac89d903d43a598397b16b02b800e_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133c2ac89d903d43a598397b16b02b800e_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:00,252 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/22a808d60527464ea65479371a62dfe5, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:00,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/22a808d60527464ea65479371a62dfe5 is 175, key is test_row_0/A:col10/1731525839413/Put/seqid=0 2024-11-13T19:24:00,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742012_1188 (size=39749) 2024-11-13T19:24:00,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:24:00,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:00,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:00,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525900442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525900452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,455 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525900453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,456 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525900461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,614 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/871c4215992b4acabb86bcfef4867879 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/871c4215992b4acabb86bcfef4867879 2024-11-13T19:24:00,623 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/C of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 871c4215992b4acabb86bcfef4867879(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:00,623 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:00,623 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/C, priority=13, startTime=1731525839586; duration=0sec 2024-11-13T19:24:00,623 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:00,623 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:C 2024-11-13T19:24:00,672 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/22a808d60527464ea65479371a62dfe5 2024-11-13T19:24:00,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/1826bc71a18341dfaecfda78b0f9f5cb is 50, key is test_row_0/B:col10/1731525839413/Put/seqid=0 2024-11-13T19:24:00,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742013_1189 (size=12151) 2024-11-13T19:24:00,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:00,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:24:00,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:00,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:00,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:00,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:00,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47408 deadline: 1731525900922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,927 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in 8f3bce5c1bcfc0bd5040de3aa8d43668 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525900951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525900961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525900972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,976 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:00,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525900972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:01,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:01,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:01,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:01,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:01,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:01,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/1826bc71a18341dfaecfda78b0f9f5cb 2024-11-13T19:24:01,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/c15671abc02543e6b87efe68adb84564 is 50, key is test_row_0/C:col10/1731525839413/Put/seqid=0 2024-11-13T19:24:01,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:01,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742014_1190 (size=12151) 2024-11-13T19:24:01,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/c15671abc02543e6b87efe68adb84564 2024-11-13T19:24:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:01,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:01,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:01,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:01,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/22a808d60527464ea65479371a62dfe5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/22a808d60527464ea65479371a62dfe5 2024-11-13T19:24:01,233 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/22a808d60527464ea65479371a62dfe5, entries=200, sequenceid=193, filesize=38.8 K 2024-11-13T19:24:01,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/1826bc71a18341dfaecfda78b0f9f5cb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1826bc71a18341dfaecfda78b0f9f5cb 2024-11-13T19:24:01,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1826bc71a18341dfaecfda78b0f9f5cb, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T19:24:01,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/c15671abc02543e6b87efe68adb84564 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c15671abc02543e6b87efe68adb84564 2024-11-13T19:24:01,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c15671abc02543e6b87efe68adb84564, entries=150, sequenceid=193, filesize=11.9 K 2024-11-13T19:24:01,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1500ms, sequenceid=193, compaction requested=false 2024-11-13T19:24:01,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:01,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-13T19:24:01,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:01,379 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T19:24:01,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379236ef99c6246aabd04e39282885f5f_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525839780/Put/seqid=0 2024-11-13T19:24:01,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742015_1191 (size=12304) 2024-11-13T19:24:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,433 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379236ef99c6246aabd04e39282885f5f_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379236ef99c6246aabd04e39282885f5f_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:01,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/684bf8ed230c496d967956a663f8ae07, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:01,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/684bf8ed230c496d967956a663f8ae07 is 175, key is test_row_0/A:col10/1731525839780/Put/seqid=0 2024-11-13T19:24:01,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742016_1192 (size=31105) 2024-11-13T19:24:01,450 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/684bf8ed230c496d967956a663f8ae07 2024-11-13T19:24:01,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/1e56e68080484cddb3bc0bad475de304 is 50, key is test_row_0/B:col10/1731525839780/Put/seqid=0 2024-11-13T19:24:01,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742017_1193 (size=12151) 2024-11-13T19:24:01,479 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/1e56e68080484cddb3bc0bad475de304 2024-11-13T19:24:01,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/372ebf2429d04b9ba944774ccd93e275 is 50, key is test_row_0/C:col10/1731525839780/Put/seqid=0 2024-11-13T19:24:01,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742018_1194 (size=12151) 2024-11-13T19:24:01,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:24:01,902 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/372ebf2429d04b9ba944774ccd93e275 2024-11-13T19:24:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/684bf8ed230c496d967956a663f8ae07 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/684bf8ed230c496d967956a663f8ae07 2024-11-13T19:24:01,947 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/684bf8ed230c496d967956a663f8ae07, entries=150, sequenceid=210, filesize=30.4 K 2024-11-13T19:24:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/1e56e68080484cddb3bc0bad475de304 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1e56e68080484cddb3bc0bad475de304 2024-11-13T19:24:01,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,977 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1e56e68080484cddb3bc0bad475de304, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T19:24:01,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/372ebf2429d04b9ba944774ccd93e275 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/372ebf2429d04b9ba944774ccd93e275 2024-11-13T19:24:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:01,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,012 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/372ebf2429d04b9ba944774ccd93e275, entries=150, sequenceid=210, filesize=11.9 K 2024-11-13T19:24:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,018 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=0 B/0 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 639ms, sequenceid=210, compaction requested=true 2024-11-13T19:24:02,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:02,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:02,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-13T19:24:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-13T19:24:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-13T19:24:02,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3770 sec 2024-11-13T19:24:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,026 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.3910 sec 2024-11-13T19:24:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,173 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:24:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113878035a990ea47b3a52f1f862f106a3a_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525842171/Put/seqid=0 2024-11-13T19:24:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742020_1196 (size=24758) 2024-11-13T19:24:02,306 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:02,319 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113878035a990ea47b3a52f1f862f106a3a_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113878035a990ea47b3a52f1f862f106a3a_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:02,322 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/607ef412eb1f4b10a7b21e6f8091eee7, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:02,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/607ef412eb1f4b10a7b21e6f8091eee7 is 175, key is test_row_0/A:col10/1731525842171/Put/seqid=0 2024-11-13T19:24:02,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742019_1195 (size=74395) 2024-11-13T19:24:02,330 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=221, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/607ef412eb1f4b10a7b21e6f8091eee7 2024-11-13T19:24:02,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/c5041a2c6fad472e8b8b386714c21302 is 50, key is test_row_0/B:col10/1731525842171/Put/seqid=0 2024-11-13T19:24:02,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525902341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525902344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525902349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742021_1197 (size=12151) 2024-11-13T19:24:02,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525902361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,375 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525902472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525902473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525902473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525902481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525902681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525902682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525902683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:02,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525902707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:02,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:02,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/c5041a2c6fad472e8b8b386714c21302 2024-11-13T19:24:02,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/104679af7e804122aa5da9ffbf9b02a7 is 50, key is test_row_0/C:col10/1731525842171/Put/seqid=0 2024-11-13T19:24:02,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742022_1198 (size=12151) 2024-11-13T19:24:02,797 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/104679af7e804122aa5da9ffbf9b02a7 2024-11-13T19:24:02,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/607ef412eb1f4b10a7b21e6f8091eee7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/607ef412eb1f4b10a7b21e6f8091eee7 2024-11-13T19:24:02,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/607ef412eb1f4b10a7b21e6f8091eee7, entries=400, sequenceid=221, filesize=72.7 K 2024-11-13T19:24:02,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/c5041a2c6fad472e8b8b386714c21302 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c5041a2c6fad472e8b8b386714c21302 2024-11-13T19:24:02,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c5041a2c6fad472e8b8b386714c21302, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T19:24:02,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/104679af7e804122aa5da9ffbf9b02a7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/104679af7e804122aa5da9ffbf9b02a7 2024-11-13T19:24:02,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/104679af7e804122aa5da9ffbf9b02a7, entries=150, sequenceid=221, filesize=11.9 K 2024-11-13T19:24:02,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 658ms, sequenceid=221, compaction requested=true 2024-11-13T19:24:02,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:02,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:02,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:02,831 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:02,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:02,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:02,831 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:02,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:24:02,832 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:02,838 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 176730 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:02,838 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:02,838 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/A is initiating minor compaction (all files) 2024-11-13T19:24:02,838 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/B is initiating minor compaction (all files) 2024-11-13T19:24:02,838 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/A in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:02,839 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/B in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:02,839 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/4dc4ab21dd114af0bd1be4f544ec343a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/22a808d60527464ea65479371a62dfe5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/684bf8ed230c496d967956a663f8ae07, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/607ef412eb1f4b10a7b21e6f8091eee7] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=172.6 K 2024-11-13T19:24:02,839 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d01cef1912bf4853ad0f766f0b6e2bb5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1826bc71a18341dfaecfda78b0f9f5cb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1e56e68080484cddb3bc0bad475de304, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c5041a2c6fad472e8b8b386714c21302] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=47.8 K 2024-11-13T19:24:02,839 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:02,839 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/4dc4ab21dd114af0bd1be4f544ec343a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/22a808d60527464ea65479371a62dfe5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/684bf8ed230c496d967956a663f8ae07, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/607ef412eb1f4b10a7b21e6f8091eee7] 2024-11-13T19:24:02,839 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting d01cef1912bf4853ad0f766f0b6e2bb5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525838195 2024-11-13T19:24:02,841 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1826bc71a18341dfaecfda78b0f9f5cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731525839413 2024-11-13T19:24:02,841 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc4ab21dd114af0bd1be4f544ec343a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525838195 2024-11-13T19:24:02,842 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22a808d60527464ea65479371a62dfe5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731525839386 2024-11-13T19:24:02,842 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e56e68080484cddb3bc0bad475de304, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731525839780 2024-11-13T19:24:02,842 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c5041a2c6fad472e8b8b386714c21302, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731525842147 2024-11-13T19:24:02,842 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 684bf8ed230c496d967956a663f8ae07, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731525839780 2024-11-13T19:24:02,843 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 607ef412eb1f4b10a7b21e6f8091eee7, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731525842141 2024-11-13T19:24:02,853 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:02,857 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#B#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:02,858 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/a8e87bac1ea546e0ad4ed5a9525b47c8 is 50, key is test_row_0/B:col10/1731525842171/Put/seqid=0 2024-11-13T19:24:02,867 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411131a755f4828e64406883c7de85dd4d1ff_8f3bce5c1bcfc0bd5040de3aa8d43668 store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:02,883 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411131a755f4828e64406883c7de85dd4d1ff_8f3bce5c1bcfc0bd5040de3aa8d43668, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:02,886 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131a755f4828e64406883c7de85dd4d1ff_8f3bce5c1bcfc0bd5040de3aa8d43668 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:02,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742023_1199 (size=12663) 2024-11-13T19:24:02,920 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/a8e87bac1ea546e0ad4ed5a9525b47c8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/a8e87bac1ea546e0ad4ed5a9525b47c8 2024-11-13T19:24:02,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742024_1200 (size=4469) 2024-11-13T19:24:02,936 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#A#compaction#168 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:02,937 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/bdff06462c7d4401a8c50a8609ec7bd0 is 175, key is test_row_0/A:col10/1731525842171/Put/seqid=0 2024-11-13T19:24:02,940 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/B of 8f3bce5c1bcfc0bd5040de3aa8d43668 into a8e87bac1ea546e0ad4ed5a9525b47c8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:02,940 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:02,940 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/B, priority=12, startTime=1731525842831; duration=0sec 2024-11-13T19:24:02,941 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:02,941 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:B 2024-11-13T19:24:02,941 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:02,945 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:02,945 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/C is initiating minor compaction (all files) 2024-11-13T19:24:02,945 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/C in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:02,945 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/871c4215992b4acabb86bcfef4867879, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c15671abc02543e6b87efe68adb84564, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/372ebf2429d04b9ba944774ccd93e275, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/104679af7e804122aa5da9ffbf9b02a7] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=47.8 K 2024-11-13T19:24:02,948 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 871c4215992b4acabb86bcfef4867879, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525838195 2024-11-13T19:24:02,949 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c15671abc02543e6b87efe68adb84564, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731525839413 2024-11-13T19:24:02,950 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 372ebf2429d04b9ba944774ccd93e275, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731525839780 2024-11-13T19:24:02,951 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 104679af7e804122aa5da9ffbf9b02a7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731525842147 2024-11-13T19:24:02,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742025_1201 (size=31617) 2024-11-13T19:24:02,980 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#C#compaction#170 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:02,981 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/3ee585f201204876a925929b26e78fe6 is 50, key is test_row_0/C:col10/1731525842171/Put/seqid=0 2024-11-13T19:24:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:02,998 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:02,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T19:24:03,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742026_1202 (size=12663) 2024-11-13T19:24:03,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525903011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,024 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/3ee585f201204876a925929b26e78fe6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3ee585f201204876a925929b26e78fe6 2024-11-13T19:24:03,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525903015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525903016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,033 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/C of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 3ee585f201204876a925929b26e78fe6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:03,033 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:03,033 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/C, priority=12, startTime=1731525842831; duration=0sec 2024-11-13T19:24:03,033 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:03,033 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:C 2024-11-13T19:24:03,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525903022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e055288280f4468b6f6754c6510f60e_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525842996/Put/seqid=0 2024-11-13T19:24:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,038 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742027_1203 (size=14794) 2024-11-13T19:24:03,060 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:03,072 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131e055288280f4468b6f6754c6510f60e_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e055288280f4468b6f6754c6510f60e_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:03,074 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/da82ff5da7654fe4aa7f6bd512a4d320, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:03,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/da82ff5da7654fe4aa7f6bd512a4d320 is 175, key is test_row_0/A:col10/1731525842996/Put/seqid=0 2024-11-13T19:24:03,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742028_1204 (size=39749) 2024-11-13T19:24:03,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/da82ff5da7654fe4aa7f6bd512a4d320 2024-11-13T19:24:03,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/b4ee057ecdd4460f8842fb40d02bbe1b is 50, key is test_row_0/B:col10/1731525842996/Put/seqid=0 2024-11-13T19:24:03,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525903135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525903137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525903137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742029_1205 (size=12151) 2024-11-13T19:24:03,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/b4ee057ecdd4460f8842fb40d02bbe1b 2024-11-13T19:24:03,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/05804b8d91bd4cffb30f41c58376ab8d is 50, key is test_row_0/C:col10/1731525842996/Put/seqid=0 2024-11-13T19:24:03,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742030_1206 (size=12151) 2024-11-13T19:24:03,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525903341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525903351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525903351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,377 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/bdff06462c7d4401a8c50a8609ec7bd0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/bdff06462c7d4401a8c50a8609ec7bd0 2024-11-13T19:24:03,382 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/A of 8f3bce5c1bcfc0bd5040de3aa8d43668 into bdff06462c7d4401a8c50a8609ec7bd0(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:03,382 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:03,382 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/A, priority=12, startTime=1731525842831; duration=0sec 2024-11-13T19:24:03,382 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:03,382 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:A 2024-11-13T19:24:03,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525903541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/05804b8d91bd4cffb30f41c58376ab8d 2024-11-13T19:24:03,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/da82ff5da7654fe4aa7f6bd512a4d320 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/da82ff5da7654fe4aa7f6bd512a4d320 2024-11-13T19:24:03,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/da82ff5da7654fe4aa7f6bd512a4d320, entries=200, sequenceid=249, filesize=38.8 K 2024-11-13T19:24:03,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/b4ee057ecdd4460f8842fb40d02bbe1b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/b4ee057ecdd4460f8842fb40d02bbe1b 2024-11-13T19:24:03,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/b4ee057ecdd4460f8842fb40d02bbe1b, entries=150, sequenceid=249, filesize=11.9 K 2024-11-13T19:24:03,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/05804b8d91bd4cffb30f41c58376ab8d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/05804b8d91bd4cffb30f41c58376ab8d 2024-11-13T19:24:03,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525903652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/05804b8d91bd4cffb30f41c58376ab8d, entries=150, sequenceid=249, filesize=11.9 K 2024-11-13T19:24:03,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 661ms, sequenceid=249, compaction requested=false 2024-11-13T19:24:03,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:03,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:03,664 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:03,664 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:24:03,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343813f697eaa49f5900b245381911722_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525843662/Put/seqid=0 2024-11-13T19:24:03,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742031_1207 (size=17434) 2024-11-13T19:24:03,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525903725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525903727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-13T19:24:03,802 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:03,802 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:03,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-13T19:24:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:24:03,805 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:03,806 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:03,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:03,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525903831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525903831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:03,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:03,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:24:03,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:03,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525904041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525904041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:04,081 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111343813f697eaa49f5900b245381911722_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343813f697eaa49f5900b245381911722_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:04,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/9b5f3dff0e8f4cbca4b5dc4014d17136, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:04,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/9b5f3dff0e8f4cbca4b5dc4014d17136 is 175, key is test_row_0/A:col10/1731525843662/Put/seqid=0 2024-11-13T19:24:04,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742032_1208 (size=48539) 2024-11-13T19:24:04,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:24:04,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525904161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,165 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525904351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525904351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:04,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:04,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:24:04,488 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/9b5f3dff0e8f4cbca4b5dc4014d17136 2024-11-13T19:24:04,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/26f8667773394337a50ddb483e520c22 is 50, key is test_row_0/B:col10/1731525843662/Put/seqid=0 2024-11-13T19:24:04,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742033_1209 (size=12251) 2024-11-13T19:24:04,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525904552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,553 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in 8f3bce5c1bcfc0bd5040de3aa8d43668 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525904861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525904861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:04,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:04,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:04,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/26f8667773394337a50ddb483e520c22 2024-11-13T19:24:04,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/4d324b815535440a9b98713e6b34a6b1 is 50, key is test_row_0/C:col10/1731525843662/Put/seqid=0 2024-11-13T19:24:04,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742034_1210 (size=12251) 2024-11-13T19:24:04,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/4d324b815535440a9b98713e6b34a6b1 2024-11-13T19:24:04,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/9b5f3dff0e8f4cbca4b5dc4014d17136 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9b5f3dff0e8f4cbca4b5dc4014d17136 2024-11-13T19:24:04,921 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9b5f3dff0e8f4cbca4b5dc4014d17136, entries=250, sequenceid=262, filesize=47.4 K 2024-11-13T19:24:04,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/26f8667773394337a50ddb483e520c22 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/26f8667773394337a50ddb483e520c22 2024-11-13T19:24:04,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/26f8667773394337a50ddb483e520c22, entries=150, sequenceid=262, filesize=12.0 K 2024-11-13T19:24:04,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/4d324b815535440a9b98713e6b34a6b1 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4d324b815535440a9b98713e6b34a6b1 2024-11-13T19:24:04,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4d324b815535440a9b98713e6b34a6b1, entries=150, sequenceid=262, filesize=12.0 K 2024-11-13T19:24:04,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1272ms, sequenceid=262, compaction requested=true 2024-11-13T19:24:04,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:04,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:04,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:04,937 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:04,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:04,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:04,937 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:04,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:04,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:04,939 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119905 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:04,939 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/A is initiating minor compaction (all files) 2024-11-13T19:24:04,939 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/A in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,939 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/bdff06462c7d4401a8c50a8609ec7bd0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/da82ff5da7654fe4aa7f6bd512a4d320, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9b5f3dff0e8f4cbca4b5dc4014d17136] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=117.1 K 2024-11-13T19:24:04,939 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,939 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/bdff06462c7d4401a8c50a8609ec7bd0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/da82ff5da7654fe4aa7f6bd512a4d320, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9b5f3dff0e8f4cbca4b5dc4014d17136] 2024-11-13T19:24:04,940 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:04,940 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/B is initiating minor compaction (all files) 2024-11-13T19:24:04,940 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdff06462c7d4401a8c50a8609ec7bd0, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731525842147 2024-11-13T19:24:04,940 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/B in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:04,940 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/a8e87bac1ea546e0ad4ed5a9525b47c8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/b4ee057ecdd4460f8842fb40d02bbe1b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/26f8667773394337a50ddb483e520c22] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=36.2 K 2024-11-13T19:24:04,940 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting da82ff5da7654fe4aa7f6bd512a4d320, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731525842311 2024-11-13T19:24:04,940 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e87bac1ea546e0ad4ed5a9525b47c8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731525842147 2024-11-13T19:24:04,941 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b5f3dff0e8f4cbca4b5dc4014d17136, keycount=250, bloomtype=ROW, size=47.4 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731525843008 2024-11-13T19:24:04,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:24:04,941 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting b4ee057ecdd4460f8842fb40d02bbe1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731525842311 2024-11-13T19:24:04,942 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f8667773394337a50ddb483e520c22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731525843011 2024-11-13T19:24:04,955 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:04,971 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113bd1a46dceff1415c9fcf22c5823e5fc3_8f3bce5c1bcfc0bd5040de3aa8d43668 store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:04,971 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:04,972 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/d08d33444d824c1295ba9a20b1ecd26e is 50, key is test_row_0/B:col10/1731525843662/Put/seqid=0 2024-11-13T19:24:04,973 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113bd1a46dceff1415c9fcf22c5823e5fc3_8f3bce5c1bcfc0bd5040de3aa8d43668, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:04,973 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bd1a46dceff1415c9fcf22c5823e5fc3_8f3bce5c1bcfc0bd5040de3aa8d43668 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742035_1211 (size=12865) 2024-11-13T19:24:04,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742036_1212 (size=4469) 2024-11-13T19:24:05,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-13T19:24:05,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:05,029 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T19:24:05,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113449e6a998fec47e29e43cd7dc959a271_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525843724/Put/seqid=0 2024-11-13T19:24:05,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742037_1213 (size=12454) 2024-11-13T19:24:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:05,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. as already flushing 2024-11-13T19:24:05,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525905198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:05,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525905312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:05,387 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#A#compaction#177 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:05,387 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/a8231b9667d2460ab184c367f63eeea7 is 175, key is test_row_0/A:col10/1731525843662/Put/seqid=0 2024-11-13T19:24:05,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742038_1214 (size=31819) 2024-11-13T19:24:05,392 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/d08d33444d824c1295ba9a20b1ecd26e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d08d33444d824c1295ba9a20b1ecd26e 2024-11-13T19:24:05,396 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/a8231b9667d2460ab184c367f63eeea7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/a8231b9667d2460ab184c367f63eeea7 2024-11-13T19:24:05,397 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/B of 8f3bce5c1bcfc0bd5040de3aa8d43668 into d08d33444d824c1295ba9a20b1ecd26e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:05,397 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:05,397 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/B, priority=13, startTime=1731525844937; duration=0sec 2024-11-13T19:24:05,397 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:05,397 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:B 2024-11-13T19:24:05,397 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:05,398 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:05,398 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/C is initiating minor compaction (all files) 2024-11-13T19:24:05,398 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/C in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:05,399 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3ee585f201204876a925929b26e78fe6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/05804b8d91bd4cffb30f41c58376ab8d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4d324b815535440a9b98713e6b34a6b1] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=36.2 K 2024-11-13T19:24:05,399 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee585f201204876a925929b26e78fe6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1731525842147 2024-11-13T19:24:05,399 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 05804b8d91bd4cffb30f41c58376ab8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731525842311 2024-11-13T19:24:05,401 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d324b815535440a9b98713e6b34a6b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731525843011 2024-11-13T19:24:05,403 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/A of 8f3bce5c1bcfc0bd5040de3aa8d43668 into a8231b9667d2460ab184c367f63eeea7(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:05,403 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:05,403 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/A, priority=13, startTime=1731525844937; duration=0sec 2024-11-13T19:24:05,403 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:05,403 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:A 2024-11-13T19:24:05,411 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:05,411 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/6e4294e7d1cb487b91e56e9d6c4ee112 is 50, key is test_row_0/C:col10/1731525843662/Put/seqid=0 2024-11-13T19:24:05,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742039_1215 (size=12865) 2024-11-13T19:24:05,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:05,446 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113449e6a998fec47e29e43cd7dc959a271_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113449e6a998fec47e29e43cd7dc959a271_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/105aa610e9d849c1b56eb4113a13c55d, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:05,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/105aa610e9d849c1b56eb4113a13c55d is 175, key is test_row_0/A:col10/1731525843724/Put/seqid=0 2024-11-13T19:24:05,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742040_1216 (size=31255) 2024-11-13T19:24:05,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:05,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525905521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:05,829 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/6e4294e7d1cb487b91e56e9d6c4ee112 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/6e4294e7d1cb487b91e56e9d6c4ee112 2024-11-13T19:24:05,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:05,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525905831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:05,834 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/C of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 6e4294e7d1cb487b91e56e9d6c4ee112(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:05,834 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:05,834 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/C, priority=13, startTime=1731525844937; duration=0sec 2024-11-13T19:24:05,834 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:05,834 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:C 2024-11-13T19:24:05,853 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=287, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/105aa610e9d849c1b56eb4113a13c55d 2024-11-13T19:24:05,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/c6f364c900824002826531758601a5d8 is 50, key is test_row_0/B:col10/1731525843724/Put/seqid=0 2024-11-13T19:24:05,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742041_1217 (size=12301) 2024-11-13T19:24:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47396 deadline: 1731525905871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47416 deadline: 1731525905871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:05,874 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in 8f3bce5c1bcfc0bd5040de3aa8d43668 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:05,874 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in 8f3bce5c1bcfc0bd5040de3aa8d43668 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:24:06,268 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/c6f364c900824002826531758601a5d8 2024-11-13T19:24:06,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/8fe591c28e7c453fbbc70707a156f528 is 50, key is test_row_0/C:col10/1731525843724/Put/seqid=0 2024-11-13T19:24:06,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742042_1218 (size=12301) 2024-11-13T19:24:06,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47376 deadline: 1731525906341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:06,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:06,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47388 deadline: 1731525906571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:06,572 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in 8f3bce5c1bcfc0bd5040de3aa8d43668 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:06,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:06,681 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/8fe591c28e7c453fbbc70707a156f528 2024-11-13T19:24:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/105aa610e9d849c1b56eb4113a13c55d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/105aa610e9d849c1b56eb4113a13c55d 2024-11-13T19:24:06,689 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/105aa610e9d849c1b56eb4113a13c55d, entries=150, sequenceid=287, filesize=30.5 K 2024-11-13T19:24:06,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/c6f364c900824002826531758601a5d8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c6f364c900824002826531758601a5d8 2024-11-13T19:24:06,694 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c6f364c900824002826531758601a5d8, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:24:06,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/8fe591c28e7c453fbbc70707a156f528 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/8fe591c28e7c453fbbc70707a156f528 2024-11-13T19:24:06,699 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/8fe591c28e7c453fbbc70707a156f528, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:24:06,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1671ms, sequenceid=287, compaction requested=false 2024-11-13T19:24:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-13T19:24:06,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-13T19:24:06,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-13T19:24:06,704 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8960 sec 2024-11-13T19:24:06,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.9020 sec 2024-11-13T19:24:06,875 INFO [Thread-695 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-695. 2024-11-13T19:24:06,875 DEBUG [Thread-695 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:06,875 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,875 DEBUG [Thread-695 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,875 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:06,877 INFO [Thread-701 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-701. 2024-11-13T19:24:06,877 DEBUG [Thread-701 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:06,877 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,877 DEBUG [Thread-701 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:06,877 INFO [Thread-697 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-697. 2024-11-13T19:24:06,877 DEBUG [Thread-697 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:06,877 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,877 DEBUG [Thread-697 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,877 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:06,879 INFO [Thread-699 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-699. 2024-11-13T19:24:06,879 DEBUG [Thread-699 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:06,879 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,879 DEBUG [Thread-699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:06,879 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:07,364 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:07,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T19:24:07,364 INFO [Thread-692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-692. 2024-11-13T19:24:07,364 DEBUG [Thread-692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:07,364 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:07,364 DEBUG [Thread-692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:07,364 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:07,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f72895ba7664b9d877398180de3a131_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525847362/Put/seqid=0 2024-11-13T19:24:07,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742043_1219 (size=12454) 2024-11-13T19:24:07,777 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:07,785 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136f72895ba7664b9d877398180de3a131_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f72895ba7664b9d877398180de3a131_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:07,787 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/7b0b29605f8e4db293d580c629e8080c, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:07,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/7b0b29605f8e4db293d580c629e8080c is 175, key is test_row_0/A:col10/1731525847362/Put/seqid=0 2024-11-13T19:24:07,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742044_1220 (size=31255) 2024-11-13T19:24:07,887 INFO [Thread-684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-684. 2024-11-13T19:24:07,887 DEBUG [Thread-684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:07,887 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:07,887 DEBUG [Thread-684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:07,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:07,904 INFO [Thread-686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-686. 2024-11-13T19:24:07,904 DEBUG [Thread-686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:07,904 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:07,904 DEBUG [Thread-686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:07,904 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:07,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-13T19:24:07,963 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:08,194 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/7b0b29605f8e4db293d580c629e8080c 2024-11-13T19:24:08,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e8267d093d1746499d9048ff6d653f78 is 50, key is test_row_0/B:col10/1731525847362/Put/seqid=0 2024-11-13T19:24:08,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742045_1221 (size=12301) 2024-11-13T19:24:08,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e8267d093d1746499d9048ff6d653f78 2024-11-13T19:24:08,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/c555f24cce244a4f9316318976cf2a51 is 50, key is test_row_0/C:col10/1731525847362/Put/seqid=0 2024-11-13T19:24:08,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742046_1222 (size=12301) 2024-11-13T19:24:09,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/c555f24cce244a4f9316318976cf2a51 2024-11-13T19:24:09,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/7b0b29605f8e4db293d580c629e8080c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/7b0b29605f8e4db293d580c629e8080c 2024-11-13T19:24:09,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/7b0b29605f8e4db293d580c629e8080c, entries=150, sequenceid=302, filesize=30.5 K 2024-11-13T19:24:09,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e8267d093d1746499d9048ff6d653f78 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e8267d093d1746499d9048ff6d653f78 2024-11-13T19:24:09,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e8267d093d1746499d9048ff6d653f78, entries=150, sequenceid=302, filesize=12.0 K 2024-11-13T19:24:09,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/c555f24cce244a4f9316318976cf2a51 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c555f24cce244a4f9316318976cf2a51 2024-11-13T19:24:09,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c555f24cce244a4f9316318976cf2a51, entries=150, sequenceid=302, filesize=12.0 K 2024-11-13T19:24:09,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=13.42 KB/13740 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1689ms, sequenceid=302, compaction requested=true 2024-11-13T19:24:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:09,053 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:09,053 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8f3bce5c1bcfc0bd5040de3aa8d43668:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:09,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:09,054 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94329 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:09,054 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:09,054 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/A is initiating minor compaction (all files) 2024-11-13T19:24:09,054 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/B is initiating minor compaction (all files) 2024-11-13T19:24:09,054 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/B in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:09,054 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/A in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:09,055 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/a8231b9667d2460ab184c367f63eeea7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/105aa610e9d849c1b56eb4113a13c55d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/7b0b29605f8e4db293d580c629e8080c] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=92.1 K 2024-11-13T19:24:09,055 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d08d33444d824c1295ba9a20b1ecd26e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c6f364c900824002826531758601a5d8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e8267d093d1746499d9048ff6d653f78] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=36.6 K 2024-11-13T19:24:09,055 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:09,055 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/a8231b9667d2460ab184c367f63eeea7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/105aa610e9d849c1b56eb4113a13c55d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/7b0b29605f8e4db293d580c629e8080c] 2024-11-13T19:24:09,055 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting d08d33444d824c1295ba9a20b1ecd26e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731525843011 2024-11-13T19:24:09,055 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8231b9667d2460ab184c367f63eeea7, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731525843011 2024-11-13T19:24:09,055 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c6f364c900824002826531758601a5d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731525843718 2024-11-13T19:24:09,055 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 105aa610e9d849c1b56eb4113a13c55d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731525843718 2024-11-13T19:24:09,056 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e8267d093d1746499d9048ff6d653f78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731525845177 2024-11-13T19:24:09,056 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0b29605f8e4db293d580c629e8080c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731525845177 2024-11-13T19:24:09,067 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:09,068 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#B#compaction#186 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:09,068 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e0c2516fc4584ac1a342b693da35e521 is 50, key is test_row_0/B:col10/1731525847362/Put/seqid=0 2024-11-13T19:24:09,069 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ca468329b56149768a1928462708c277_8f3bce5c1bcfc0bd5040de3aa8d43668 store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:09,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742047_1223 (size=13017) 2024-11-13T19:24:09,091 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ca468329b56149768a1928462708c277_8f3bce5c1bcfc0bd5040de3aa8d43668, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:09,092 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ca468329b56149768a1928462708c277_8f3bce5c1bcfc0bd5040de3aa8d43668 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:09,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742048_1224 (size=4469) 2024-11-13T19:24:09,483 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/e0c2516fc4584ac1a342b693da35e521 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e0c2516fc4584ac1a342b693da35e521 2024-11-13T19:24:09,489 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/B of 8f3bce5c1bcfc0bd5040de3aa8d43668 into e0c2516fc4584ac1a342b693da35e521(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:09,489 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:09,489 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/B, priority=13, startTime=1731525849053; duration=0sec 2024-11-13T19:24:09,489 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:09,489 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:B 2024-11-13T19:24:09,490 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:09,491 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:09,491 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): 8f3bce5c1bcfc0bd5040de3aa8d43668/C is initiating minor compaction (all files) 2024-11-13T19:24:09,491 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8f3bce5c1bcfc0bd5040de3aa8d43668/C in TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:09,492 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/6e4294e7d1cb487b91e56e9d6c4ee112, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/8fe591c28e7c453fbbc70707a156f528, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c555f24cce244a4f9316318976cf2a51] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp, totalSize=36.6 K 2024-11-13T19:24:09,492 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e4294e7d1cb487b91e56e9d6c4ee112, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731525843011 2024-11-13T19:24:09,492 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fe591c28e7c453fbbc70707a156f528, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731525843718 2024-11-13T19:24:09,493 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c555f24cce244a4f9316318976cf2a51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1731525845177 2024-11-13T19:24:09,498 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:09,499 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/8e89f715f0264b07b3d485a5b30e2d41 is 175, key is test_row_0/A:col10/1731525847362/Put/seqid=0 2024-11-13T19:24:09,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742049_1225 (size=31971) 2024-11-13T19:24:09,503 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8f3bce5c1bcfc0bd5040de3aa8d43668#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:09,504 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/1daf3be641ee445794b914ffdaa88262 is 50, key is test_row_0/C:col10/1731525847362/Put/seqid=0 2024-11-13T19:24:09,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742050_1226 (size=13017) 2024-11-13T19:24:09,909 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/8e89f715f0264b07b3d485a5b30e2d41 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8e89f715f0264b07b3d485a5b30e2d41 2024-11-13T19:24:09,913 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/1daf3be641ee445794b914ffdaa88262 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/1daf3be641ee445794b914ffdaa88262 2024-11-13T19:24:09,915 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/A of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 8e89f715f0264b07b3d485a5b30e2d41(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:09,915 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:09,915 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/A, priority=13, startTime=1731525849053; duration=0sec 2024-11-13T19:24:09,915 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:09,915 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:A 2024-11-13T19:24:09,919 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8f3bce5c1bcfc0bd5040de3aa8d43668/C of 8f3bce5c1bcfc0bd5040de3aa8d43668 into 1daf3be641ee445794b914ffdaa88262(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:09,919 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: 2024-11-13T19:24:09,919 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., storeName=8f3bce5c1bcfc0bd5040de3aa8d43668/C, priority=13, startTime=1731525849053; duration=0sec 2024-11-13T19:24:09,919 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:09,919 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8f3bce5c1bcfc0bd5040de3aa8d43668:C 2024-11-13T19:24:10,596 INFO [Thread-688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-688. 2024-11-13T19:24:10,596 DEBUG [Thread-688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:10,596 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:10,596 DEBUG [Thread-688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:10,596 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:10,751 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:24:11,023 INFO [Thread-690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-690. 2024-11-13T19:24:11,023 DEBUG [Thread-690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:11,023 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:11,023 DEBUG [Thread-690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:11,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3340 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3364 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1484 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4450 rows 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1489 2024-11-13T19:24:11,023 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4464 rows 2024-11-13T19:24:11,023 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:24:11,023 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:24:11,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:11,024 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:11,024 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:11,024 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:24:11,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:11,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:24:11,028 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525851027"}]},"ts":"1731525851027"} 2024-11-13T19:24:11,029 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:24:11,029 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:24:11,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:24:11,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, UNASSIGN}] 2024-11-13T19:24:11,033 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, UNASSIGN 2024-11-13T19:24:11,034 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:11,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, UNASSIGN because future has completed 2024-11-13T19:24:11,035 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:24:11,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:24:11,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:24:11,189 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:11,190 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:24:11,190 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 8f3bce5c1bcfc0bd5040de3aa8d43668, disabling compactions & flushes 2024-11-13T19:24:11,190 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:11,190 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:11,190 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. after waiting 0 ms 2024-11-13T19:24:11,190 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:11,191 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T19:24:11,202 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ca45ea093f5a4ff9bec33879c0c10a07_8f3bce5c1bcfc0bd5040de3aa8d43668 is 50, key is test_row_0/A:col10/1731525847902/Put/seqid=0 2024-11-13T19:24:11,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742051_1227 (size=12454) 2024-11-13T19:24:11,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:24:11,608 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:11,617 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ca45ea093f5a4ff9bec33879c0c10a07_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca45ea093f5a4ff9bec33879c0c10a07_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:11,619 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0e09c086a205443d8a208b5c2dc28672, store: [table=TestAcidGuarantees family=A region=8f3bce5c1bcfc0bd5040de3aa8d43668] 2024-11-13T19:24:11,619 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0e09c086a205443d8a208b5c2dc28672 is 175, key is test_row_0/A:col10/1731525847902/Put/seqid=0 2024-11-13T19:24:11,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742052_1228 (size=31255) 2024-11-13T19:24:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:24:12,027 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0e09c086a205443d8a208b5c2dc28672 2024-11-13T19:24:12,040 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/6e354cffbd3f48b683d36c78635b7bce is 50, key is test_row_0/B:col10/1731525847902/Put/seqid=0 2024-11-13T19:24:12,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742053_1229 (size=12301) 2024-11-13T19:24:12,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:24:12,446 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/6e354cffbd3f48b683d36c78635b7bce 2024-11-13T19:24:12,460 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/3705ace3dcc5411e9a96411fb5649ccf is 50, key is test_row_0/C:col10/1731525847902/Put/seqid=0 2024-11-13T19:24:12,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742054_1230 (size=12301) 2024-11-13T19:24:12,865 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/3705ace3dcc5411e9a96411fb5649ccf 2024-11-13T19:24:12,899 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/A/0e09c086a205443d8a208b5c2dc28672 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0e09c086a205443d8a208b5c2dc28672 2024-11-13T19:24:12,909 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0e09c086a205443d8a208b5c2dc28672, entries=150, sequenceid=312, filesize=30.5 K 2024-11-13T19:24:12,910 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/B/6e354cffbd3f48b683d36c78635b7bce as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/6e354cffbd3f48b683d36c78635b7bce 2024-11-13T19:24:12,915 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/6e354cffbd3f48b683d36c78635b7bce, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T19:24:12,916 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/.tmp/C/3705ace3dcc5411e9a96411fb5649ccf as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3705ace3dcc5411e9a96411fb5649ccf 2024-11-13T19:24:12,921 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3705ace3dcc5411e9a96411fb5649ccf, entries=150, sequenceid=312, filesize=12.0 K 2024-11-13T19:24:12,922 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1732ms, sequenceid=312, compaction requested=false 2024-11-13T19:24:12,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8058a37e334842c099f9b616e290834b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/27af637effe24da78dc3079fabaa8a54, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/f2410070e26a4bb38dfdfc165359c7ff, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/cef17d35eb434aeb989a0d569db0e4d3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/59dc5c6762954880ae37fed346099528, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9cdec6f41dee4b8b800751bbd6dea0b9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/c37bc0e54f21408487b3843eb819757b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/51438133451e4fe19da6b6e0034fe8ad, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0f2b7d8fb8d64963b02f2fb65a08a8db, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/b36c75bde3154299a34c0cbec786cf25, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/05f565096e3b457f9ccb467e5d31807d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/4dc4ab21dd114af0bd1be4f544ec343a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/22a808d60527464ea65479371a62dfe5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/684bf8ed230c496d967956a663f8ae07, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/607ef412eb1f4b10a7b21e6f8091eee7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/bdff06462c7d4401a8c50a8609ec7bd0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/da82ff5da7654fe4aa7f6bd512a4d320, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9b5f3dff0e8f4cbca4b5dc4014d17136, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/a8231b9667d2460ab184c367f63eeea7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/105aa610e9d849c1b56eb4113a13c55d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/7b0b29605f8e4db293d580c629e8080c] to archive 2024-11-13T19:24:12,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:24:12,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8058a37e334842c099f9b616e290834b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8058a37e334842c099f9b616e290834b 2024-11-13T19:24:12,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/27af637effe24da78dc3079fabaa8a54 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/27af637effe24da78dc3079fabaa8a54 2024-11-13T19:24:12,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/f2410070e26a4bb38dfdfc165359c7ff to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/f2410070e26a4bb38dfdfc165359c7ff 2024-11-13T19:24:12,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/cef17d35eb434aeb989a0d569db0e4d3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/cef17d35eb434aeb989a0d569db0e4d3 2024-11-13T19:24:12,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/59dc5c6762954880ae37fed346099528 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/59dc5c6762954880ae37fed346099528 2024-11-13T19:24:12,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9cdec6f41dee4b8b800751bbd6dea0b9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9cdec6f41dee4b8b800751bbd6dea0b9 2024-11-13T19:24:12,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/c37bc0e54f21408487b3843eb819757b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/c37bc0e54f21408487b3843eb819757b 2024-11-13T19:24:12,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/51438133451e4fe19da6b6e0034fe8ad to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/51438133451e4fe19da6b6e0034fe8ad 2024-11-13T19:24:12,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0f2b7d8fb8d64963b02f2fb65a08a8db to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0f2b7d8fb8d64963b02f2fb65a08a8db 2024-11-13T19:24:12,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/b36c75bde3154299a34c0cbec786cf25 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/b36c75bde3154299a34c0cbec786cf25 2024-11-13T19:24:12,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/05f565096e3b457f9ccb467e5d31807d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/05f565096e3b457f9ccb467e5d31807d 2024-11-13T19:24:12,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/4dc4ab21dd114af0bd1be4f544ec343a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/4dc4ab21dd114af0bd1be4f544ec343a 2024-11-13T19:24:12,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/22a808d60527464ea65479371a62dfe5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/22a808d60527464ea65479371a62dfe5 2024-11-13T19:24:12,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/684bf8ed230c496d967956a663f8ae07 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/684bf8ed230c496d967956a663f8ae07 2024-11-13T19:24:12,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/607ef412eb1f4b10a7b21e6f8091eee7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/607ef412eb1f4b10a7b21e6f8091eee7 2024-11-13T19:24:12,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/bdff06462c7d4401a8c50a8609ec7bd0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/bdff06462c7d4401a8c50a8609ec7bd0 2024-11-13T19:24:12,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/da82ff5da7654fe4aa7f6bd512a4d320 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/da82ff5da7654fe4aa7f6bd512a4d320 2024-11-13T19:24:12,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9b5f3dff0e8f4cbca4b5dc4014d17136 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/9b5f3dff0e8f4cbca4b5dc4014d17136 2024-11-13T19:24:12,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/a8231b9667d2460ab184c367f63eeea7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/a8231b9667d2460ab184c367f63eeea7 2024-11-13T19:24:12,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/105aa610e9d849c1b56eb4113a13c55d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/105aa610e9d849c1b56eb4113a13c55d 2024-11-13T19:24:12,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/7b0b29605f8e4db293d580c629e8080c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/7b0b29605f8e4db293d580c629e8080c 2024-11-13T19:24:12,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/4e944e4bf41f403eb8af27734d200ff3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/154d3b5823234cb996114ef9f5b3a022, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/cb5baed9dfbc4a37834cb2f342ca73bd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/de4473c19db8488a9fd8513a5dfbf742, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/61551e3c396b4f9d824399c524fdb2ab, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/f2e2712a6c3449ef88427e572afac53b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/83e171fb36ef4e21bf83668ba27d0584, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/88c6ef2e7a2944dea80020e29d8b3b11, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/8764c02ff4564d4ea25909eb55a78c16, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/2639756fcc1f43ccac2f8263d63202f3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d01cef1912bf4853ad0f766f0b6e2bb5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e00b4b3c701c41588ccd73d0f248d09d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1826bc71a18341dfaecfda78b0f9f5cb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1e56e68080484cddb3bc0bad475de304, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/a8e87bac1ea546e0ad4ed5a9525b47c8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c5041a2c6fad472e8b8b386714c21302, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/b4ee057ecdd4460f8842fb40d02bbe1b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d08d33444d824c1295ba9a20b1ecd26e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/26f8667773394337a50ddb483e520c22, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c6f364c900824002826531758601a5d8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e8267d093d1746499d9048ff6d653f78] to archive 2024-11-13T19:24:12,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:24:12,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/4e944e4bf41f403eb8af27734d200ff3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/4e944e4bf41f403eb8af27734d200ff3 2024-11-13T19:24:12,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/154d3b5823234cb996114ef9f5b3a022 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/154d3b5823234cb996114ef9f5b3a022 2024-11-13T19:24:12,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/cb5baed9dfbc4a37834cb2f342ca73bd to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/cb5baed9dfbc4a37834cb2f342ca73bd 2024-11-13T19:24:12,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/de4473c19db8488a9fd8513a5dfbf742 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/de4473c19db8488a9fd8513a5dfbf742 2024-11-13T19:24:12,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/61551e3c396b4f9d824399c524fdb2ab to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/61551e3c396b4f9d824399c524fdb2ab 2024-11-13T19:24:12,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/f2e2712a6c3449ef88427e572afac53b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/f2e2712a6c3449ef88427e572afac53b 2024-11-13T19:24:12,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/83e171fb36ef4e21bf83668ba27d0584 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/83e171fb36ef4e21bf83668ba27d0584 2024-11-13T19:24:12,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/88c6ef2e7a2944dea80020e29d8b3b11 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/88c6ef2e7a2944dea80020e29d8b3b11 2024-11-13T19:24:12,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/8764c02ff4564d4ea25909eb55a78c16 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/8764c02ff4564d4ea25909eb55a78c16 2024-11-13T19:24:12,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/2639756fcc1f43ccac2f8263d63202f3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/2639756fcc1f43ccac2f8263d63202f3 2024-11-13T19:24:12,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d01cef1912bf4853ad0f766f0b6e2bb5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d01cef1912bf4853ad0f766f0b6e2bb5 2024-11-13T19:24:12,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e00b4b3c701c41588ccd73d0f248d09d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e00b4b3c701c41588ccd73d0f248d09d 2024-11-13T19:24:12,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1826bc71a18341dfaecfda78b0f9f5cb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1826bc71a18341dfaecfda78b0f9f5cb 2024-11-13T19:24:12,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1e56e68080484cddb3bc0bad475de304 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/1e56e68080484cddb3bc0bad475de304 2024-11-13T19:24:12,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/a8e87bac1ea546e0ad4ed5a9525b47c8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/a8e87bac1ea546e0ad4ed5a9525b47c8 2024-11-13T19:24:12,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c5041a2c6fad472e8b8b386714c21302 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c5041a2c6fad472e8b8b386714c21302 2024-11-13T19:24:12,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/b4ee057ecdd4460f8842fb40d02bbe1b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/b4ee057ecdd4460f8842fb40d02bbe1b 2024-11-13T19:24:12,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d08d33444d824c1295ba9a20b1ecd26e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/d08d33444d824c1295ba9a20b1ecd26e 2024-11-13T19:24:12,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/26f8667773394337a50ddb483e520c22 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/26f8667773394337a50ddb483e520c22 2024-11-13T19:24:12,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c6f364c900824002826531758601a5d8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/c6f364c900824002826531758601a5d8 2024-11-13T19:24:12,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e8267d093d1746499d9048ff6d653f78 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e8267d093d1746499d9048ff6d653f78 2024-11-13T19:24:12,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/f44743be15d24f42849ad47b9d2f1aa7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/14c0b6add5fb48dcaaa802201f9efed7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/97048f9d704746c1b7b10b54f65c9bec, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4f1d8cc83ba74148b963951cc3f81e34, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9601188f9b4849449675cef67ff26710, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9de7e9aaeff841d7bf044f8216859149, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/caf3e000ac01462f8e416917a9260db9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/2359a7aa59a4479b8fa709c4a108a4df, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/03da0c6c8edf4ddea07db56b3bd92f45, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/367a0faa656345c9ab46eb49a108cefe, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/871c4215992b4acabb86bcfef4867879, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/61b72302450346fda32e9327789ebb99, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c15671abc02543e6b87efe68adb84564, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/372ebf2429d04b9ba944774ccd93e275, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3ee585f201204876a925929b26e78fe6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/104679af7e804122aa5da9ffbf9b02a7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/05804b8d91bd4cffb30f41c58376ab8d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/6e4294e7d1cb487b91e56e9d6c4ee112, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4d324b815535440a9b98713e6b34a6b1, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/8fe591c28e7c453fbbc70707a156f528, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c555f24cce244a4f9316318976cf2a51] to archive 2024-11-13T19:24:12,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:24:12,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/f44743be15d24f42849ad47b9d2f1aa7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/f44743be15d24f42849ad47b9d2f1aa7 2024-11-13T19:24:12,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/14c0b6add5fb48dcaaa802201f9efed7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/14c0b6add5fb48dcaaa802201f9efed7 2024-11-13T19:24:12,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/97048f9d704746c1b7b10b54f65c9bec to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/97048f9d704746c1b7b10b54f65c9bec 2024-11-13T19:24:12,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4f1d8cc83ba74148b963951cc3f81e34 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4f1d8cc83ba74148b963951cc3f81e34 2024-11-13T19:24:12,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9601188f9b4849449675cef67ff26710 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9601188f9b4849449675cef67ff26710 2024-11-13T19:24:12,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9de7e9aaeff841d7bf044f8216859149 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/9de7e9aaeff841d7bf044f8216859149 2024-11-13T19:24:12,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/caf3e000ac01462f8e416917a9260db9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/caf3e000ac01462f8e416917a9260db9 2024-11-13T19:24:12,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/2359a7aa59a4479b8fa709c4a108a4df to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/2359a7aa59a4479b8fa709c4a108a4df 2024-11-13T19:24:12,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/03da0c6c8edf4ddea07db56b3bd92f45 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/03da0c6c8edf4ddea07db56b3bd92f45 2024-11-13T19:24:12,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/367a0faa656345c9ab46eb49a108cefe to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/367a0faa656345c9ab46eb49a108cefe 2024-11-13T19:24:12,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/871c4215992b4acabb86bcfef4867879 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/871c4215992b4acabb86bcfef4867879 2024-11-13T19:24:12,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/61b72302450346fda32e9327789ebb99 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/61b72302450346fda32e9327789ebb99 2024-11-13T19:24:12,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c15671abc02543e6b87efe68adb84564 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c15671abc02543e6b87efe68adb84564 2024-11-13T19:24:12,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/372ebf2429d04b9ba944774ccd93e275 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/372ebf2429d04b9ba944774ccd93e275 2024-11-13T19:24:12,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3ee585f201204876a925929b26e78fe6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3ee585f201204876a925929b26e78fe6 2024-11-13T19:24:12,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/104679af7e804122aa5da9ffbf9b02a7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/104679af7e804122aa5da9ffbf9b02a7 2024-11-13T19:24:12,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/05804b8d91bd4cffb30f41c58376ab8d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/05804b8d91bd4cffb30f41c58376ab8d 2024-11-13T19:24:12,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/6e4294e7d1cb487b91e56e9d6c4ee112 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/6e4294e7d1cb487b91e56e9d6c4ee112 2024-11-13T19:24:12,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4d324b815535440a9b98713e6b34a6b1 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/4d324b815535440a9b98713e6b34a6b1 2024-11-13T19:24:12,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/8fe591c28e7c453fbbc70707a156f528 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/8fe591c28e7c453fbbc70707a156f528 2024-11-13T19:24:12,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c555f24cce244a4f9316318976cf2a51 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/c555f24cce244a4f9316318976cf2a51 2024-11-13T19:24:12,999 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/recovered.edits/315.seqid, newMaxSeqId=315, maxSeqId=4 2024-11-13T19:24:13,000 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. 2024-11-13T19:24:13,000 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 8f3bce5c1bcfc0bd5040de3aa8d43668: Waiting for close lock at 1731525851190Running coprocessor pre-close hooks at 1731525851190Disabling compacts and flushes for region at 1731525851190Disabling writes for close at 1731525851190Obtaining lock to block concurrent updates at 1731525851191 (+1 ms)Preparing flush snapshotting stores in 8f3bce5c1bcfc0bd5040de3aa8d43668 at 1731525851191Finished memstore snapshotting TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731525851192 (+1 ms)Flushing stores of TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668. at 1731525851193 (+1 ms)Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/A: creating writer at 1731525851193Flushing mob file 8f3bce5c1bcfc0bd5040de3aa8d43668/A: appending metadata at 1731525851202 (+9 ms)Flushing mob file 8f3bce5c1bcfc0bd5040de3aa8d43668/A: closing flushed file at 1731525851202Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/A: appending metadata at 1731525851619 (+417 ms)Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/A: closing flushed file at 1731525851619Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/B: creating writer at 1731525852036 (+417 ms)Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/B: appending metadata at 1731525852039 (+3 ms)Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/B: closing flushed file at 1731525852039Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/C: creating writer at 1731525852455 (+416 ms)Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/C: appending metadata at 1731525852459 (+4 ms)Flushing 8f3bce5c1bcfc0bd5040de3aa8d43668/C: closing flushed file at 1731525852460 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1bb706ef: reopening flushed file at 1731525852897 (+437 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ff5f6b2: reopening flushed file at 1731525852909 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23df950e: reopening flushed file at 1731525852915 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 8f3bce5c1bcfc0bd5040de3aa8d43668 in 1732ms, sequenceid=312, compaction requested=false at 1731525852922 (+7 ms)Writing region close event to WAL at 1731525852996 (+74 ms)Running coprocessor post-close hooks at 1731525853000 (+4 ms)Closed at 1731525853000 2024-11-13T19:24:13,002 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=8f3bce5c1bcfc0bd5040de3aa8d43668, regionState=CLOSED 2024-11-13T19:24:13,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:24:13,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T19:24:13,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 8f3bce5c1bcfc0bd5040de3aa8d43668, server=e4b56bc8c4b3,40371,1731525792389 in 1.9690 sec 2024-11-13T19:24:13,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-13T19:24:13,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8f3bce5c1bcfc0bd5040de3aa8d43668, UNASSIGN in 1.9740 sec 2024-11-13T19:24:13,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-13T19:24:13,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9780 sec 2024-11-13T19:24:13,013 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525853013"}]},"ts":"1731525853013"} 2024-11-13T19:24:13,014 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:24:13,014 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:24:13,016 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9910 sec 2024-11-13T19:24:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-13T19:24:13,173 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:13,174 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:24:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:13,178 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:24:13,179 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:13,183 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,187 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/recovered.edits] 2024-11-13T19:24:13,192 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0e09c086a205443d8a208b5c2dc28672 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/0e09c086a205443d8a208b5c2dc28672 2024-11-13T19:24:13,194 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8e89f715f0264b07b3d485a5b30e2d41 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/A/8e89f715f0264b07b3d485a5b30e2d41 2024-11-13T19:24:13,198 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/6e354cffbd3f48b683d36c78635b7bce to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/6e354cffbd3f48b683d36c78635b7bce 2024-11-13T19:24:13,199 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e0c2516fc4584ac1a342b693da35e521 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/B/e0c2516fc4584ac1a342b693da35e521 2024-11-13T19:24:13,202 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/1daf3be641ee445794b914ffdaa88262 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/1daf3be641ee445794b914ffdaa88262 2024-11-13T19:24:13,203 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3705ace3dcc5411e9a96411fb5649ccf to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/C/3705ace3dcc5411e9a96411fb5649ccf 2024-11-13T19:24:13,207 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/recovered.edits/315.seqid to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668/recovered.edits/315.seqid 2024-11-13T19:24:13,207 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,207 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:24:13,208 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:24:13,209 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T19:24:13,213 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ae887df912d4727b6658dcee063b5bb_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411130ae887df912d4727b6658dcee063b5bb_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,215 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131bce309084ba45df939f65eb5a48689b_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131bce309084ba45df939f65eb5a48689b_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,216 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e055288280f4468b6f6754c6510f60e_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131e055288280f4468b6f6754c6510f60e_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,217 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133c2ac89d903d43a598397b16b02b800e_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133c2ac89d903d43a598397b16b02b800e_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,219 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343813f697eaa49f5900b245381911722_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111343813f697eaa49f5900b245381911722_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,220 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113449e6a998fec47e29e43cd7dc959a271_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113449e6a998fec47e29e43cd7dc959a271_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,221 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134e7a7600ade144769181c6c19ee90efb_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411134e7a7600ade144769181c6c19ee90efb_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,222 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f72895ba7664b9d877398180de3a131_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136f72895ba7664b9d877398180de3a131_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,223 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371a4bd5e16f5463bbaa149be86350809_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111371a4bd5e16f5463bbaa149be86350809_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,224 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111378f727f8839445e39ed70ac3e9508ba8_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111378f727f8839445e39ed70ac3e9508ba8_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,225 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379236ef99c6246aabd04e39282885f5f_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379236ef99c6246aabd04e39282885f5f_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,226 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111380752e1546f6447ca185b5eae539456a_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111380752e1546f6447ca185b5eae539456a_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,227 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113878035a990ea47b3a52f1f862f106a3a_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113878035a990ea47b3a52f1f862f106a3a_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,228 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a44faba4cf114029b26f3746570a93be_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a44faba4cf114029b26f3746570a93be_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,229 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca45ea093f5a4ff9bec33879c0c10a07_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ca45ea093f5a4ff9bec33879c0c10a07_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,230 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cba611fc9cab4264a190142ad768747a_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113cba611fc9cab4264a190142ad768747a_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,231 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee0f0ba43bf7409ba60c70291065d327_8f3bce5c1bcfc0bd5040de3aa8d43668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113ee0f0ba43bf7409ba60c70291065d327_8f3bce5c1bcfc0bd5040de3aa8d43668 2024-11-13T19:24:13,231 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:24:13,233 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:13,235 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:24:13,237 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:24:13,238 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:13,238 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:24:13,238 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731525853238"}]},"ts":"9223372036854775807"} 2024-11-13T19:24:13,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:24:13,240 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8f3bce5c1bcfc0bd5040de3aa8d43668, NAME => 'TestAcidGuarantees,,1731525823694.8f3bce5c1bcfc0bd5040de3aa8d43668.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:24:13,240 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:24:13,240 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731525853240"}]},"ts":"9223372036854775807"} 2024-11-13T19:24:13,242 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:24:13,243 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:13,244 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-13T19:24:13,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-13T19:24:13,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:24:13,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:13,295 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349642852_22 at /127.0.0.1:35694 [Waiting for operation #113] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-340608075_22 at /127.0.0.1:60526 [Waiting for operation #610] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-340608075_22 at /127.0.0.1:35682 [Waiting for operation #143] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-349642852_22 at /127.0.0.1:38852 [Waiting for operation #573] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1677 (was 359) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=238 (was 1536) 2024-11-13T19:24:13,306 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=1677, ProcessCount=11, AvailableMemoryMB=238 2024-11-13T19:24:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:24:13,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:24:13,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:13,311 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:24:13,311 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:13,311 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-13T19:24:13,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-13T19:24:13,312 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:24:13,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742055_1231 (size=959) 2024-11-13T19:24:13,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-13T19:24:13,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-13T19:24:13,725 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:24:13,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742056_1232 (size=53) 2024-11-13T19:24:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-13T19:24:14,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:24:14,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b9ac03ed4dfd24d768bc705ca3d577cd, disabling compactions & flushes 2024-11-13T19:24:14,138 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:14,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:14,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. after waiting 0 ms 2024-11-13T19:24:14,138 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:14,138 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:14,139 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b9ac03ed4dfd24d768bc705ca3d577cd: Waiting for close lock at 1731525854138Disabling compacts and flushes for region at 1731525854138Disabling writes for close at 1731525854138Writing region close event to WAL at 1731525854138Closed at 1731525854138 2024-11-13T19:24:14,140 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:24:14,140 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731525854140"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731525854140"}]},"ts":"1731525854140"} 2024-11-13T19:24:14,142 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:24:14,143 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:24:14,143 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525854143"}]},"ts":"1731525854143"} 2024-11-13T19:24:14,145 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:24:14,145 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, ASSIGN}] 2024-11-13T19:24:14,147 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, ASSIGN 2024-11-13T19:24:14,148 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, ASSIGN; state=OFFLINE, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=false 2024-11-13T19:24:14,300 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b9ac03ed4dfd24d768bc705ca3d577cd, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:14,304 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, ASSIGN because future has completed 2024-11-13T19:24:14,305 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:24:14,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-13T19:24:14,461 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:14,461 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:24:14,461 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,461 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:24:14,461 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,461 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,463 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,464 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9ac03ed4dfd24d768bc705ca3d577cd columnFamilyName A 2024-11-13T19:24:14,464 DEBUG [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:14,465 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] regionserver.HStore(327): Store=b9ac03ed4dfd24d768bc705ca3d577cd/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:14,465 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,466 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9ac03ed4dfd24d768bc705ca3d577cd columnFamilyName B 2024-11-13T19:24:14,466 DEBUG [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:14,466 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] regionserver.HStore(327): Store=b9ac03ed4dfd24d768bc705ca3d577cd/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:14,466 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,467 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b9ac03ed4dfd24d768bc705ca3d577cd columnFamilyName C 2024-11-13T19:24:14,467 DEBUG [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:14,468 INFO [StoreOpener-b9ac03ed4dfd24d768bc705ca3d577cd-1 {}] regionserver.HStore(327): Store=b9ac03ed4dfd24d768bc705ca3d577cd/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:14,468 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,469 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,469 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,470 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,470 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,470 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:24:14,471 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,473 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:24:14,473 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened b9ac03ed4dfd24d768bc705ca3d577cd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69849245, jitterRate=0.040834859013557434}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:24:14,473 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:14,474 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for b9ac03ed4dfd24d768bc705ca3d577cd: Running coprocessor pre-open hook at 1731525854462Writing region info on filesystem at 1731525854462Initializing all the Stores at 1731525854462Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525854462Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525854463 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525854463Cleaning up temporary data from old regions at 1731525854470 (+7 ms)Running coprocessor post-open hooks at 1731525854473 (+3 ms)Region opened successfully at 1731525854474 (+1 ms) 2024-11-13T19:24:14,475 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., pid=63, masterSystemTime=1731525854458 2024-11-13T19:24:14,477 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:14,477 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:14,477 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=b9ac03ed4dfd24d768bc705ca3d577cd, regionState=OPEN, openSeqNum=2, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:14,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:24:14,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T19:24:14,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 in 174 msec 2024-11-13T19:24:14,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-13T19:24:14,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, ASSIGN in 336 msec 2024-11-13T19:24:14,483 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:24:14,484 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525854483"}]},"ts":"1731525854483"} 2024-11-13T19:24:14,485 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:24:14,486 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:24:14,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-13T19:24:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-13T19:24:15,462 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:15,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,466 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@378b1f57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19f3ba15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,470 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,473 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,474 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:24:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:24:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@28a8bf55 2024-11-13T19:24:15,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:24:15,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:24:15,479 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,479 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,479 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,480 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51339a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,482 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64dc25cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,483 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,483 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,484 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d95fc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,484 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,485 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ec1458d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,486 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,486 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,486 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,487 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,487 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d4140a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,488 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,488 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@95deed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,490 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,490 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,490 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,491 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b3eeb1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,491 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,492 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fd8ef43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,496 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@169ca00c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,497 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54866, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15195332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,499 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,499 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a8695f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,501 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@570c34a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,502 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ed310c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,502 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,502 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,503 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57226193, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f832ccf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@176a9725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,506 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c3ef883, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,507 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130bce22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5762fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,509 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357902ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78035968, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f72a925, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:15,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:15,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:15,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f70b2d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:15,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:15,513 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:15,514 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6624a387, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:15,514 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,514 DEBUG [Thread-1090 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,514 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,514 DEBUG [Thread-1086 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:15,515 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,515 DEBUG [Thread-1088 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,516 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,516 DEBUG [Thread-1094 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-13T19:24:15,517 DEBUG [Thread-1092 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,517 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,519 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:15,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,519 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,520 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:15,521 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:15,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:24:15,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,524 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:15,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:15,526 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:15,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:24:15,528 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:15,528 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T19:24:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525915546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525915547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525915547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/94400ab5df65412c9b4d9090d04dd7e7 is 50, key is test_row_1/A:col10/1731525855526/Put/seqid=0 2024-11-13T19:24:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525915550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525915549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742057_1233 (size=9657) 2024-11-13T19:24:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:24:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525915661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525915661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,663 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525915661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525915662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525915661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:24:15,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:15,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:15,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:24:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:15,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:24:15,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525915871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525915871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525915871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525915871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525915871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:15,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/94400ab5df65412c9b4d9090d04dd7e7 2024-11-13T19:24:15,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:24:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:15,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:15,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/0d28c8832ff945dca76bc1001eb24cef is 50, key is test_row_1/B:col10/1731525855526/Put/seqid=0 2024-11-13T19:24:15,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742058_1234 (size=9657) 2024-11-13T19:24:15,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/0d28c8832ff945dca76bc1001eb24cef 2024-11-13T19:24:16,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/acae3fc3d54549d780aee6544ff56269 is 50, key is test_row_1/C:col10/1731525855526/Put/seqid=0 2024-11-13T19:24:16,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742059_1235 (size=9657) 2024-11-13T19:24:16,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:24:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:16,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:24:16,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525916181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525916181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525916181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525916182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525916182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,185 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:24:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:16,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/acae3fc3d54549d780aee6544ff56269 2024-11-13T19:24:16,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/94400ab5df65412c9b4d9090d04dd7e7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/94400ab5df65412c9b4d9090d04dd7e7 2024-11-13T19:24:16,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:24:16,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:16,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:16,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:16,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/94400ab5df65412c9b4d9090d04dd7e7, entries=100, sequenceid=13, filesize=9.4 K 2024-11-13T19:24:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:16,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/0d28c8832ff945dca76bc1001eb24cef as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0d28c8832ff945dca76bc1001eb24cef 2024-11-13T19:24:16,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0d28c8832ff945dca76bc1001eb24cef, entries=100, sequenceid=13, filesize=9.4 K 2024-11-13T19:24:16,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/acae3fc3d54549d780aee6544ff56269 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/acae3fc3d54549d780aee6544ff56269 2024-11-13T19:24:16,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/acae3fc3d54549d780aee6544ff56269, entries=100, sequenceid=13, filesize=9.4 K 2024-11-13T19:24:16,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b9ac03ed4dfd24d768bc705ca3d577cd in 927ms, sequenceid=13, compaction requested=false 2024-11-13T19:24:16,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:16,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-13T19:24:16,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:16,592 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:24:16,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/3b7589ac571848d08902281e2cb3f3db is 50, key is test_row_0/A:col10/1731525855546/Put/seqid=0 2024-11-13T19:24:16,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742060_1236 (size=12001) 2024-11-13T19:24:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:24:16,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:16,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525916696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525916699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525916700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525916701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,705 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525916701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525916812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525916812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525916812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525916812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525916813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:16,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,007 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/3b7589ac571848d08902281e2cb3f3db 2024-11-13T19:24:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1f3503e89f7a4ab9a932b4ff12bed8ce is 50, key is test_row_0/B:col10/1731525855546/Put/seqid=0 2024-11-13T19:24:17,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525917022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525917022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525917022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525917022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525917031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742061_1237 (size=12001) 2024-11-13T19:24:17,047 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1f3503e89f7a4ab9a932b4ff12bed8ce 2024-11-13T19:24:17,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/702637a31c054d5ea595e5711d9336a2 is 50, key is test_row_0/C:col10/1731525855546/Put/seqid=0 2024-11-13T19:24:17,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742062_1238 (size=12001) 2024-11-13T19:24:17,085 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/702637a31c054d5ea595e5711d9336a2 2024-11-13T19:24:17,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/3b7589ac571848d08902281e2cb3f3db as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3b7589ac571848d08902281e2cb3f3db 2024-11-13T19:24:17,100 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3b7589ac571848d08902281e2cb3f3db, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:24:17,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1f3503e89f7a4ab9a932b4ff12bed8ce as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1f3503e89f7a4ab9a932b4ff12bed8ce 2024-11-13T19:24:17,113 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1f3503e89f7a4ab9a932b4ff12bed8ce, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:24:17,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/702637a31c054d5ea595e5711d9336a2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/702637a31c054d5ea595e5711d9336a2 2024-11-13T19:24:17,126 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/702637a31c054d5ea595e5711d9336a2, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:24:17,128 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b9ac03ed4dfd24d768bc705ca3d577cd in 537ms, sequenceid=37, compaction requested=false 2024-11-13T19:24:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:17,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-13T19:24:17,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-13T19:24:17,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-13T19:24:17,136 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6110 sec 2024-11-13T19:24:17,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6220 sec 2024-11-13T19:24:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:17,337 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:17,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T19:24:17,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/2e6767a0212342c28ee13ece095d4e54 is 50, key is test_row_0/A:col10/1731525857336/Put/seqid=0 2024-11-13T19:24:17,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742063_1239 (size=12001) 2024-11-13T19:24:17,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525917392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525917387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525917397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525917400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525917408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525917511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525917511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,519 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525917522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525917522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525917532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-13T19:24:17,672 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:17,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-13T19:24:17,675 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:17,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:24:17,676 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:17,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:17,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525917722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525917732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525917733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525917733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,739 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:17,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525917742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:17,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:24:17,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/2e6767a0212342c28ee13ece095d4e54 2024-11-13T19:24:17,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/054e0463d388439cb6f586fc0b87169d is 50, key is test_row_0/B:col10/1731525857336/Put/seqid=0 2024-11-13T19:24:17,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:24:17,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:17,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:17,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:17,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742064_1240 (size=12001) 2024-11-13T19:24:17,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/054e0463d388439cb6f586fc0b87169d 2024-11-13T19:24:17,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/c6500d1798e74717af0dff93472aecec is 50, key is test_row_0/C:col10/1731525857336/Put/seqid=0 2024-11-13T19:24:17,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742065_1241 (size=12001) 2024-11-13T19:24:17,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:24:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:17,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:17,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:17,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:17,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:17,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:24:18,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525918032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525918042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525918042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525918052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,056 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525918052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:24:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:18,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,139 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:18,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:18,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:18,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:24:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:18,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:18,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:24:18,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/c6500d1798e74717af0dff93472aecec 2024-11-13T19:24:18,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/2e6767a0212342c28ee13ece095d4e54 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/2e6767a0212342c28ee13ece095d4e54 2024-11-13T19:24:18,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/2e6767a0212342c28ee13ece095d4e54, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T19:24:18,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/054e0463d388439cb6f586fc0b87169d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/054e0463d388439cb6f586fc0b87169d 2024-11-13T19:24:18,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/054e0463d388439cb6f586fc0b87169d, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T19:24:18,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/c6500d1798e74717af0dff93472aecec as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c6500d1798e74717af0dff93472aecec 2024-11-13T19:24:18,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c6500d1798e74717af0dff93472aecec, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T19:24:18,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for b9ac03ed4dfd24d768bc705ca3d577cd in 1028ms, sequenceid=52, compaction requested=true 2024-11-13T19:24:18,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:18,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:18,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:18,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:18,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-13T19:24:18,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:18,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-13T19:24:18,367 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:18,368 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:18,369 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:18,369 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:18,369 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/C is initiating minor compaction (all files) 2024-11-13T19:24:18,369 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/A is initiating minor compaction (all files) 2024-11-13T19:24:18,369 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/C in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,369 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/A in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,369 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/94400ab5df65412c9b4d9090d04dd7e7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3b7589ac571848d08902281e2cb3f3db, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/2e6767a0212342c28ee13ece095d4e54] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=32.9 K 2024-11-13T19:24:18,369 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/acae3fc3d54549d780aee6544ff56269, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/702637a31c054d5ea595e5711d9336a2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c6500d1798e74717af0dff93472aecec] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=32.9 K 2024-11-13T19:24:18,370 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 94400ab5df65412c9b4d9090d04dd7e7, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731525855526 2024-11-13T19:24:18,370 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting acae3fc3d54549d780aee6544ff56269, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731525855526 2024-11-13T19:24:18,370 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b7589ac571848d08902281e2cb3f3db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525855544 2024-11-13T19:24:18,371 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e6767a0212342c28ee13ece095d4e54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525857335 2024-11-13T19:24:18,371 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 702637a31c054d5ea595e5711d9336a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525855544 2024-11-13T19:24:18,372 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6500d1798e74717af0dff93472aecec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525857335 2024-11-13T19:24:18,385 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:18,385 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/d6773754a334412e9f90a251a2370cde is 50, key is test_row_0/A:col10/1731525857336/Put/seqid=0 2024-11-13T19:24:18,388 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#C#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:18,389 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/6474133ea7ba45d6bb94ec4f7e206f9f is 50, key is test_row_0/C:col10/1731525857336/Put/seqid=0 2024-11-13T19:24:18,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742066_1242 (size=12104) 2024-11-13T19:24:18,439 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/d6773754a334412e9f90a251a2370cde as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/d6773754a334412e9f90a251a2370cde 2024-11-13T19:24:18,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742067_1243 (size=12104) 2024-11-13T19:24:18,455 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/A of b9ac03ed4dfd24d768bc705ca3d577cd into d6773754a334412e9f90a251a2370cde(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:18,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:18,455 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/A, priority=13, startTime=1731525858366; duration=0sec 2024-11-13T19:24:18,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:18,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:A 2024-11-13T19:24:18,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:18,458 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:18,458 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/B is initiating minor compaction (all files) 2024-11-13T19:24:18,458 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/B in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,459 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0d28c8832ff945dca76bc1001eb24cef, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1f3503e89f7a4ab9a932b4ff12bed8ce, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/054e0463d388439cb6f586fc0b87169d] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=32.9 K 2024-11-13T19:24:18,460 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d28c8832ff945dca76bc1001eb24cef, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731525855526 2024-11-13T19:24:18,460 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/6474133ea7ba45d6bb94ec4f7e206f9f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/6474133ea7ba45d6bb94ec4f7e206f9f 2024-11-13T19:24:18,460 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3503e89f7a4ab9a932b4ff12bed8ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525855544 2024-11-13T19:24:18,461 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 054e0463d388439cb6f586fc0b87169d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525857335 2024-11-13T19:24:18,471 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/C of b9ac03ed4dfd24d768bc705ca3d577cd into 6474133ea7ba45d6bb94ec4f7e206f9f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:18,471 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:18,471 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/C, priority=13, startTime=1731525858366; duration=0sec 2024-11-13T19:24:18,474 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:18,474 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:C 2024-11-13T19:24:18,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-13T19:24:18,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:18,482 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T19:24:18,486 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#B#compaction#203 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:18,486 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/7ed13f62b690481caf54b631e47e7136 is 50, key is test_row_0/B:col10/1731525857336/Put/seqid=0 2024-11-13T19:24:18,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/49c42cb5d1c143ecbb7f803979b8c405 is 50, key is test_row_0/A:col10/1731525857397/Put/seqid=0 2024-11-13T19:24:18,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742069_1245 (size=12001) 2024-11-13T19:24:18,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742068_1244 (size=12104) 2024-11-13T19:24:18,536 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:24:18,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525918592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525918594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525918598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525918599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525918599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525918711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525918712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525918712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525918712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525918712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:24:18,921 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/49c42cb5d1c143ecbb7f803979b8c405 2024-11-13T19:24:18,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525918921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,925 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525918921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525918921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/0e9c20d1698545379e47431f10837655 is 50, key is test_row_0/B:col10/1731525857397/Put/seqid=0 2024-11-13T19:24:18,935 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/7ed13f62b690481caf54b631e47e7136 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7ed13f62b690481caf54b631e47e7136 2024-11-13T19:24:18,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525918932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525918932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:18,945 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/B of b9ac03ed4dfd24d768bc705ca3d577cd into 7ed13f62b690481caf54b631e47e7136(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:18,945 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:18,945 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/B, priority=13, startTime=1731525858366; duration=0sec 2024-11-13T19:24:18,945 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:18,946 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:B 2024-11-13T19:24:18,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742070_1246 (size=12001) 2024-11-13T19:24:19,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525919232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525919232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525919232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525919241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525919251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,367 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/0e9c20d1698545379e47431f10837655 2024-11-13T19:24:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/261739c59234453b8a3a592367a2bea8 is 50, key is test_row_0/C:col10/1731525857397/Put/seqid=0 2024-11-13T19:24:19,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742071_1247 (size=12001) 2024-11-13T19:24:19,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525919742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525919742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525919742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525919751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,754 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525919761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:19,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:19,812 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/261739c59234453b8a3a592367a2bea8 2024-11-13T19:24:19,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/49c42cb5d1c143ecbb7f803979b8c405 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/49c42cb5d1c143ecbb7f803979b8c405 2024-11-13T19:24:19,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:24:19,827 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/49c42cb5d1c143ecbb7f803979b8c405, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T19:24:19,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/0e9c20d1698545379e47431f10837655 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0e9c20d1698545379e47431f10837655 2024-11-13T19:24:19,837 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0e9c20d1698545379e47431f10837655, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T19:24:19,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/261739c59234453b8a3a592367a2bea8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/261739c59234453b8a3a592367a2bea8 2024-11-13T19:24:19,849 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/261739c59234453b8a3a592367a2bea8, entries=150, sequenceid=75, filesize=11.7 K 2024-11-13T19:24:19,851 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for b9ac03ed4dfd24d768bc705ca3d577cd in 1369ms, sequenceid=75, compaction requested=false 2024-11-13T19:24:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:19,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-13T19:24:19,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-13T19:24:19,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-13T19:24:19,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-11-13T19:24:19,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.1890 sec 2024-11-13T19:24:20,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:20,753 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:20,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T19:24:20,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/90e5e4d26ad449a29cfda8e3cd3e9da4 is 50, key is test_row_0/A:col10/1731525858593/Put/seqid=0 2024-11-13T19:24:20,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742072_1248 (size=16681) 2024-11-13T19:24:20,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525920772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525920773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525920773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525920776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525920782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,783 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525920881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525920882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525920882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525920882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525921091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525921092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525921092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,094 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525921092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,095 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/90e5e4d26ad449a29cfda8e3cd3e9da4 2024-11-13T19:24:21,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/5fea97962ace476aa84d7759032a3fec is 50, key is test_row_0/B:col10/1731525858593/Put/seqid=0 2024-11-13T19:24:21,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742073_1249 (size=12001) 2024-11-13T19:24:21,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525921401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525921401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525921401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525921402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,578 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/5fea97962ace476aa84d7759032a3fec 2024-11-13T19:24:21,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/480c0d6655d24fb898ef1ae333b306f5 is 50, key is test_row_0/C:col10/1731525858593/Put/seqid=0 2024-11-13T19:24:21,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742074_1250 (size=12001) 2024-11-13T19:24:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-13T19:24:21,842 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:21,842 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-13T19:24:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:24:21,844 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:21,845 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:21,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:21,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525921911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525921911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525921911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525921912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:21,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:24:21,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:24:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:21,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:21,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:22,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/480c0d6655d24fb898ef1ae333b306f5 2024-11-13T19:24:22,015 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/90e5e4d26ad449a29cfda8e3cd3e9da4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/90e5e4d26ad449a29cfda8e3cd3e9da4 2024-11-13T19:24:22,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/90e5e4d26ad449a29cfda8e3cd3e9da4, entries=250, sequenceid=94, filesize=16.3 K 2024-11-13T19:24:22,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/5fea97962ace476aa84d7759032a3fec as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5fea97962ace476aa84d7759032a3fec 2024-11-13T19:24:22,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5fea97962ace476aa84d7759032a3fec, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T19:24:22,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/480c0d6655d24fb898ef1ae333b306f5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/480c0d6655d24fb898ef1ae333b306f5 2024-11-13T19:24:22,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/480c0d6655d24fb898ef1ae333b306f5, entries=150, sequenceid=94, filesize=11.7 K 2024-11-13T19:24:22,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b9ac03ed4dfd24d768bc705ca3d577cd in 1275ms, sequenceid=94, compaction requested=true 2024-11-13T19:24:22,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:22,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:22,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:22,029 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:22,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:22,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:22,029 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:22,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:22,029 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:22,030 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:22,030 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:22,030 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/B is initiating minor compaction (all files) 2024-11-13T19:24:22,030 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/A is initiating minor compaction (all files) 2024-11-13T19:24:22,030 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/B in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:22,030 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/A in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:22,030 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/d6773754a334412e9f90a251a2370cde, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/49c42cb5d1c143ecbb7f803979b8c405, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/90e5e4d26ad449a29cfda8e3cd3e9da4] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=39.8 K 2024-11-13T19:24:22,030 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7ed13f62b690481caf54b631e47e7136, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0e9c20d1698545379e47431f10837655, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5fea97962ace476aa84d7759032a3fec] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=35.3 K 2024-11-13T19:24:22,031 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed13f62b690481caf54b631e47e7136, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525857335 2024-11-13T19:24:22,031 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6773754a334412e9f90a251a2370cde, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525857335 2024-11-13T19:24:22,031 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49c42cb5d1c143ecbb7f803979b8c405, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731525857391 2024-11-13T19:24:22,031 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e9c20d1698545379e47431f10837655, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731525857391 2024-11-13T19:24:22,031 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fea97962ace476aa84d7759032a3fec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525858593 2024-11-13T19:24:22,031 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90e5e4d26ad449a29cfda8e3cd3e9da4, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525858587 2024-11-13T19:24:22,040 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:22,040 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:22,040 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/583e527368344d33b67aebb5f17d3359 is 50, key is test_row_0/A:col10/1731525858593/Put/seqid=0 2024-11-13T19:24:22,041 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/03e2dcf61b634ca38a8be440a3a0734b is 50, key is test_row_0/B:col10/1731525858593/Put/seqid=0 2024-11-13T19:24:22,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742075_1251 (size=12207) 2024-11-13T19:24:22,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742076_1252 (size=12207) 2024-11-13T19:24:22,066 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/583e527368344d33b67aebb5f17d3359 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/583e527368344d33b67aebb5f17d3359 2024-11-13T19:24:22,071 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/A of b9ac03ed4dfd24d768bc705ca3d577cd into 583e527368344d33b67aebb5f17d3359(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:22,071 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:22,071 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/A, priority=13, startTime=1731525862029; duration=0sec 2024-11-13T19:24:22,071 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:22,071 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:A 2024-11-13T19:24:22,071 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:22,072 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:22,073 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/C is initiating minor compaction (all files) 2024-11-13T19:24:22,073 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/C in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:22,073 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/6474133ea7ba45d6bb94ec4f7e206f9f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/261739c59234453b8a3a592367a2bea8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/480c0d6655d24fb898ef1ae333b306f5] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=35.3 K 2024-11-13T19:24:22,073 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6474133ea7ba45d6bb94ec4f7e206f9f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525857335 2024-11-13T19:24:22,073 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 261739c59234453b8a3a592367a2bea8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731525857391 2024-11-13T19:24:22,074 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 480c0d6655d24fb898ef1ae333b306f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525858593 2024-11-13T19:24:22,081 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:22,082 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/59d12492a92949f386426163ef789ea7 is 50, key is test_row_0/C:col10/1731525858593/Put/seqid=0 2024-11-13T19:24:22,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742077_1253 (size=12207) 2024-11-13T19:24:22,101 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/59d12492a92949f386426163ef789ea7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/59d12492a92949f386426163ef789ea7 2024-11-13T19:24:22,106 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/C of b9ac03ed4dfd24d768bc705ca3d577cd into 59d12492a92949f386426163ef789ea7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:22,106 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:22,106 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/C, priority=13, startTime=1731525862029; duration=0sec 2024-11-13T19:24:22,106 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:22,106 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:C 2024-11-13T19:24:22,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-13T19:24:22,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:22,151 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T19:24:22,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6a0f672faab74325b1630d42fad222be is 50, key is test_row_0/A:col10/1731525860775/Put/seqid=0 2024-11-13T19:24:22,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742078_1254 (size=12001) 2024-11-13T19:24:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:24:22,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/03e2dcf61b634ca38a8be440a3a0734b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/03e2dcf61b634ca38a8be440a3a0734b 2024-11-13T19:24:22,459 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/B of b9ac03ed4dfd24d768bc705ca3d577cd into 03e2dcf61b634ca38a8be440a3a0734b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:22,459 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:22,459 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/B, priority=13, startTime=1731525862029; duration=0sec 2024-11-13T19:24:22,459 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:22,459 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:B 2024-11-13T19:24:22,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:24:22,560 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6a0f672faab74325b1630d42fad222be 2024-11-13T19:24:22,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/3a8e6c272f4746f9a5bd8dbba1873f05 is 50, key is test_row_0/B:col10/1731525860775/Put/seqid=0 2024-11-13T19:24:22,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742079_1255 (size=12001) 2024-11-13T19:24:22,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:22,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:22,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525922854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:22,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:22,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525922922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525922922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:22,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:22,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525922922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:22,925 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:22,926 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,925 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2154 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:22,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525922931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:22,934 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:22,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525922961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:22,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:22,972 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/3a8e6c272f4746f9a5bd8dbba1873f05 2024-11-13T19:24:22,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/0481e62875af43b5a28b693bc27f8d46 is 50, key is test_row_0/C:col10/1731525860775/Put/seqid=0 2024-11-13T19:24:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:24:22,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742080_1256 (size=12001) 2024-11-13T19:24:23,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:23,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525923171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:23,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:23,387 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/0481e62875af43b5a28b693bc27f8d46 2024-11-13T19:24:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6a0f672faab74325b1630d42fad222be as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6a0f672faab74325b1630d42fad222be 2024-11-13T19:24:23,395 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6a0f672faab74325b1630d42fad222be, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:24:23,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/3a8e6c272f4746f9a5bd8dbba1873f05 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3a8e6c272f4746f9a5bd8dbba1873f05 2024-11-13T19:24:23,400 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3a8e6c272f4746f9a5bd8dbba1873f05, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:24:23,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/0481e62875af43b5a28b693bc27f8d46 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/0481e62875af43b5a28b693bc27f8d46 2024-11-13T19:24:23,406 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/0481e62875af43b5a28b693bc27f8d46, entries=150, sequenceid=114, filesize=11.7 K 2024-11-13T19:24:23,407 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b9ac03ed4dfd24d768bc705ca3d577cd in 1257ms, sequenceid=114, compaction requested=false 2024-11-13T19:24:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:23,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-13T19:24:23,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-13T19:24:23,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-13T19:24:23,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5640 sec 2024-11-13T19:24:23,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.5690 sec 2024-11-13T19:24:23,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:23,483 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:23,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T19:24:23,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/4388f545e9fc4fe0977c2e7d8cc88f98 is 50, key is test_row_0/A:col10/1731525862850/Put/seqid=0 2024-11-13T19:24:23,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742081_1257 (size=12151) 2024-11-13T19:24:23,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:23,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525923536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:23,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525923641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:23,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:23,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525923851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:23,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:23,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/4388f545e9fc4fe0977c2e7d8cc88f98 2024-11-13T19:24:23,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/69061f33195542ed960c3a9f8b946fb8 is 50, key is test_row_0/B:col10/1731525862850/Put/seqid=0 2024-11-13T19:24:23,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742082_1258 (size=12151) 2024-11-13T19:24:24,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-13T19:24:24,002 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:24,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:24,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-13T19:24:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:24:24,005 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:24,005 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:24,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:24:24,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:24:24,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:24,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:24,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525924161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:24,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:24:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/69061f33195542ed960c3a9f8b946fb8 2024-11-13T19:24:24,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/fe2a1b1535ae41dbaab6d8628e564cd5 is 50, key is test_row_0/C:col10/1731525862850/Put/seqid=0 2024-11-13T19:24:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:24:24,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742083_1259 (size=12151) 2024-11-13T19:24:24,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:24:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:24,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:24:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:24:24,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525924671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:24,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/fe2a1b1535ae41dbaab6d8628e564cd5 2024-11-13T19:24:24,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/4388f545e9fc4fe0977c2e7d8cc88f98 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4388f545e9fc4fe0977c2e7d8cc88f98 2024-11-13T19:24:24,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4388f545e9fc4fe0977c2e7d8cc88f98, entries=150, sequenceid=134, filesize=11.9 K 2024-11-13T19:24:24,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/69061f33195542ed960c3a9f8b946fb8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/69061f33195542ed960c3a9f8b946fb8 2024-11-13T19:24:24,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/69061f33195542ed960c3a9f8b946fb8, entries=150, sequenceid=134, filesize=11.9 K 2024-11-13T19:24:24,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/fe2a1b1535ae41dbaab6d8628e564cd5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/fe2a1b1535ae41dbaab6d8628e564cd5 2024-11-13T19:24:24,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/fe2a1b1535ae41dbaab6d8628e564cd5, entries=150, sequenceid=134, filesize=11.9 K 2024-11-13T19:24:24,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b9ac03ed4dfd24d768bc705ca3d577cd in 1267ms, sequenceid=134, compaction requested=true 2024-11-13T19:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:24,750 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:24,750 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:24,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:24,751 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:24,751 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:24,751 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/A is initiating minor compaction (all files) 2024-11-13T19:24:24,751 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/B is initiating minor compaction (all files) 2024-11-13T19:24:24,751 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/B in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,751 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/A in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,752 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/583e527368344d33b67aebb5f17d3359, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6a0f672faab74325b1630d42fad222be, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4388f545e9fc4fe0977c2e7d8cc88f98] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=35.5 K 2024-11-13T19:24:24,752 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/03e2dcf61b634ca38a8be440a3a0734b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3a8e6c272f4746f9a5bd8dbba1873f05, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/69061f33195542ed960c3a9f8b946fb8] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=35.5 K 2024-11-13T19:24:24,752 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 03e2dcf61b634ca38a8be440a3a0734b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525858593 2024-11-13T19:24:24,752 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 583e527368344d33b67aebb5f17d3359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525858593 2024-11-13T19:24:24,752 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a0f672faab74325b1630d42fad222be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525860770 2024-11-13T19:24:24,752 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a8e6c272f4746f9a5bd8dbba1873f05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525860770 2024-11-13T19:24:24,752 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4388f545e9fc4fe0977c2e7d8cc88f98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731525862850 2024-11-13T19:24:24,753 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 69061f33195542ed960c3a9f8b946fb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731525862850 2024-11-13T19:24:24,760 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:24,760 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#A#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:24,760 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/01cebd83b5ca4241b0bcd11c791b95a4 is 50, key is test_row_0/B:col10/1731525862850/Put/seqid=0 2024-11-13T19:24:24,761 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/a58196d268ba4d1d9b90b0c4be01131d is 50, key is test_row_0/A:col10/1731525862850/Put/seqid=0 2024-11-13T19:24:24,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-13T19:24:24,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,770 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T19:24:24,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742085_1261 (size=12459) 2024-11-13T19:24:24,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6e5286a9c5824960942c088e1dc9cdbf is 50, key is test_row_0/A:col10/1731525863535/Put/seqid=0 2024-11-13T19:24:24,782 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/01cebd83b5ca4241b0bcd11c791b95a4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/01cebd83b5ca4241b0bcd11c791b95a4 2024-11-13T19:24:24,787 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/B of b9ac03ed4dfd24d768bc705ca3d577cd into 01cebd83b5ca4241b0bcd11c791b95a4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:24,787 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:24,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742086_1262 (size=12151) 2024-11-13T19:24:24,788 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/B, priority=13, startTime=1731525864750; duration=0sec 2024-11-13T19:24:24,788 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:24,788 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:B 2024-11-13T19:24:24,788 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:24,789 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:24,789 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/C is initiating minor compaction (all files) 2024-11-13T19:24:24,789 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/C in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:24,790 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/59d12492a92949f386426163ef789ea7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/0481e62875af43b5a28b693bc27f8d46, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/fe2a1b1535ae41dbaab6d8628e564cd5] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=35.5 K 2024-11-13T19:24:24,790 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d12492a92949f386426163ef789ea7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731525858593 2024-11-13T19:24:24,790 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0481e62875af43b5a28b693bc27f8d46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731525860770 2024-11-13T19:24:24,790 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting fe2a1b1535ae41dbaab6d8628e564cd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731525862850 2024-11-13T19:24:24,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742084_1260 (size=12459) 2024-11-13T19:24:24,798 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#C#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:24,798 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/d1ff1302faca4bcfb4aa1692903e7f58 is 50, key is test_row_0/C:col10/1731525862850/Put/seqid=0 2024-11-13T19:24:24,804 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/a58196d268ba4d1d9b90b0c4be01131d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/a58196d268ba4d1d9b90b0c4be01131d 2024-11-13T19:24:24,808 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/A of b9ac03ed4dfd24d768bc705ca3d577cd into a58196d268ba4d1d9b90b0c4be01131d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:24,808 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:24,808 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/A, priority=13, startTime=1731525864750; duration=0sec 2024-11-13T19:24:24,809 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:24,809 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:A 2024-11-13T19:24:24,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742087_1263 (size=12459) 2024-11-13T19:24:24,817 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/d1ff1302faca4bcfb4aa1692903e7f58 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/d1ff1302faca4bcfb4aa1692903e7f58 2024-11-13T19:24:24,823 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/C of b9ac03ed4dfd24d768bc705ca3d577cd into d1ff1302faca4bcfb4aa1692903e7f58(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:24,823 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:24,823 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/C, priority=13, startTime=1731525864750; duration=0sec 2024-11-13T19:24:24,823 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:24,823 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:C 2024-11-13T19:24:24,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:24,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525924967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525924967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:24,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525924968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525924968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:24,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525925081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525925081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525925082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525925082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:24:25,188 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6e5286a9c5824960942c088e1dc9cdbf 2024-11-13T19:24:25,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083 is 50, key is test_row_0/B:col10/1731525863535/Put/seqid=0 2024-11-13T19:24:25,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742088_1264 (size=12151) 2024-11-13T19:24:25,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525925291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525925291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525925291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525925293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,600 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083 2024-11-13T19:24:25,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525925602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525925602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525925602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525925602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/bc7c51df5d1642d7a33afbe42a1815e0 is 50, key is test_row_0/C:col10/1731525863535/Put/seqid=0 2024-11-13T19:24:25,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742089_1265 (size=12151) 2024-11-13T19:24:25,624 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/bc7c51df5d1642d7a33afbe42a1815e0 2024-11-13T19:24:25,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6e5286a9c5824960942c088e1dc9cdbf as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e5286a9c5824960942c088e1dc9cdbf 2024-11-13T19:24:25,637 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e5286a9c5824960942c088e1dc9cdbf, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T19:24:25,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083 2024-11-13T19:24:25,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T19:24:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/bc7c51df5d1642d7a33afbe42a1815e0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/bc7c51df5d1642d7a33afbe42a1815e0 2024-11-13T19:24:25,649 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/bc7c51df5d1642d7a33afbe42a1815e0, entries=150, sequenceid=151, filesize=11.9 K 2024-11-13T19:24:25,650 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b9ac03ed4dfd24d768bc705ca3d577cd in 880ms, sequenceid=151, compaction requested=false 2024-11-13T19:24:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:25,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-13T19:24:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-13T19:24:25,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-13T19:24:25,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6460 sec 2024-11-13T19:24:25,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.6520 sec 2024-11-13T19:24:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:25,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T19:24:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:25,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6df411c531ad449fb596c46bae0d2c92 is 50, key is test_row_0/A:col10/1731525864966/Put/seqid=0 2024-11-13T19:24:25,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742090_1266 (size=14541) 2024-11-13T19:24:25,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6df411c531ad449fb596c46bae0d2c92 2024-11-13T19:24:25,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/d2a8c87fed314b159ca0c4480612f75e is 50, key is test_row_0/B:col10/1731525864966/Put/seqid=0 2024-11-13T19:24:25,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742091_1267 (size=12151) 2024-11-13T19:24:25,752 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525925749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:25,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525925862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525926071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525926111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525926111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525926111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525926111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/d2a8c87fed314b159ca0c4480612f75e 2024-11-13T19:24:26,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/068b64683bf74447aa5f788537897441 is 50, key is test_row_0/C:col10/1731525864966/Put/seqid=0 2024-11-13T19:24:26,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742092_1268 (size=12151) 2024-11-13T19:24:26,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/068b64683bf74447aa5f788537897441 2024-11-13T19:24:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-13T19:24:26,162 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:26,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-13T19:24:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:24:26,165 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:26,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6df411c531ad449fb596c46bae0d2c92 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6df411c531ad449fb596c46bae0d2c92 2024-11-13T19:24:26,165 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:26,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:26,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6df411c531ad449fb596c46bae0d2c92, entries=200, sequenceid=174, filesize=14.2 K 2024-11-13T19:24:26,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/d2a8c87fed314b159ca0c4480612f75e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/d2a8c87fed314b159ca0c4480612f75e 2024-11-13T19:24:26,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/d2a8c87fed314b159ca0c4480612f75e, entries=150, sequenceid=174, filesize=11.9 K 2024-11-13T19:24:26,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/068b64683bf74447aa5f788537897441 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/068b64683bf74447aa5f788537897441 2024-11-13T19:24:26,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/068b64683bf74447aa5f788537897441, entries=150, sequenceid=174, filesize=11.9 K 2024-11-13T19:24:26,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b9ac03ed4dfd24d768bc705ca3d577cd in 501ms, sequenceid=174, compaction requested=true 2024-11-13T19:24:26,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:26,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:26,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:26,186 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:26,187 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:26,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:26,189 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:26,189 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/B is initiating minor compaction (all files) 2024-11-13T19:24:26,189 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/B in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:26,189 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/01cebd83b5ca4241b0bcd11c791b95a4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/d2a8c87fed314b159ca0c4480612f75e] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=35.9 K 2024-11-13T19:24:26,190 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:26,190 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 01cebd83b5ca4241b0bcd11c791b95a4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731525862850 2024-11-13T19:24:26,190 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/A is initiating minor compaction (all files) 2024-11-13T19:24:26,190 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/A in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:26,190 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/a58196d268ba4d1d9b90b0c4be01131d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e5286a9c5824960942c088e1dc9cdbf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6df411c531ad449fb596c46bae0d2c92] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=38.2 K 2024-11-13T19:24:26,191 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb2ebd4c7bc49fdb5e5dbd2b3dbe083, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731525863513 2024-11-13T19:24:26,191 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58196d268ba4d1d9b90b0c4be01131d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731525862850 2024-11-13T19:24:26,193 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5286a9c5824960942c088e1dc9cdbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731525863513 2024-11-13T19:24:26,193 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting d2a8c87fed314b159ca0c4480612f75e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731525864961 2024-11-13T19:24:26,194 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6df411c531ad449fb596c46bae0d2c92, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731525864961 2024-11-13T19:24:26,205 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#B#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:26,205 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/61b2fcff4cf64a3e8897a6e896b5ba6b is 50, key is test_row_0/B:col10/1731525864966/Put/seqid=0 2024-11-13T19:24:26,207 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:26,207 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/9b9ec4c799eb4096be40150e0dbd9526 is 50, key is test_row_0/A:col10/1731525864966/Put/seqid=0 2024-11-13T19:24:26,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742093_1269 (size=12561) 2024-11-13T19:24:26,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742094_1270 (size=12561) 2024-11-13T19:24:26,253 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/61b2fcff4cf64a3e8897a6e896b5ba6b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/61b2fcff4cf64a3e8897a6e896b5ba6b 2024-11-13T19:24:26,262 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/B of b9ac03ed4dfd24d768bc705ca3d577cd into 61b2fcff4cf64a3e8897a6e896b5ba6b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:26,262 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:26,262 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/B, priority=13, startTime=1731525866186; duration=0sec 2024-11-13T19:24:26,262 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:26,262 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:B 2024-11-13T19:24:26,262 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:26,264 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:26,264 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/C is initiating minor compaction (all files) 2024-11-13T19:24:26,264 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/C in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:26,264 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/d1ff1302faca4bcfb4aa1692903e7f58, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/bc7c51df5d1642d7a33afbe42a1815e0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/068b64683bf74447aa5f788537897441] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=35.9 K 2024-11-13T19:24:26,265 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ff1302faca4bcfb4aa1692903e7f58, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731525862850 2024-11-13T19:24:26,266 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting bc7c51df5d1642d7a33afbe42a1815e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731525863513 2024-11-13T19:24:26,266 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 068b64683bf74447aa5f788537897441, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731525864961 2024-11-13T19:24:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:24:26,281 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:26,282 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/8ed87107a93247b6abeeb5dc7a4a8f07 is 50, key is test_row_0/C:col10/1731525864966/Put/seqid=0 2024-11-13T19:24:26,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742095_1271 (size=12561) 2024-11-13T19:24:26,297 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/8ed87107a93247b6abeeb5dc7a4a8f07 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8ed87107a93247b6abeeb5dc7a4a8f07 2024-11-13T19:24:26,304 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/C of b9ac03ed4dfd24d768bc705ca3d577cd into 8ed87107a93247b6abeeb5dc7a4a8f07(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:26,304 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:26,304 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/C, priority=13, startTime=1731525866187; duration=0sec 2024-11-13T19:24:26,304 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:26,304 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:C 2024-11-13T19:24:26,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-13T19:24:26,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:26,319 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T19:24:26,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/3536d21e2a7e4daabf0fa80b4c05142a is 50, key is test_row_0/A:col10/1731525865742/Put/seqid=0 2024-11-13T19:24:26,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742096_1272 (size=12151) 2024-11-13T19:24:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:26,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:26,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525926447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:24:26,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525926561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,637 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/9b9ec4c799eb4096be40150e0dbd9526 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/9b9ec4c799eb4096be40150e0dbd9526 2024-11-13T19:24:26,642 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/A of b9ac03ed4dfd24d768bc705ca3d577cd into 9b9ec4c799eb4096be40150e0dbd9526(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:26,642 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:26,642 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/A, priority=13, startTime=1731525866186; duration=0sec 2024-11-13T19:24:26,642 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:26,642 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:A 2024-11-13T19:24:26,729 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/3536d21e2a7e4daabf0fa80b4c05142a 2024-11-13T19:24:26,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/fbb11c8412a9408393b89080beb5f73c is 50, key is test_row_0/B:col10/1731525865742/Put/seqid=0 2024-11-13T19:24:26,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742097_1273 (size=12151) 2024-11-13T19:24:26,740 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/fbb11c8412a9408393b89080beb5f73c 2024-11-13T19:24:26,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/063f7fb69af34729a7cf000695040359 is 50, key is test_row_0/C:col10/1731525865742/Put/seqid=0 2024-11-13T19:24:26,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742098_1274 (size=12151) 2024-11-13T19:24:26,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525926771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:26,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:26,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:24:27,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525927081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:27,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525927121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,123 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:27,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525927131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525927131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525927131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,133 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:27,134 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:27,134 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:27,152 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/063f7fb69af34729a7cf000695040359 2024-11-13T19:24:27,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/3536d21e2a7e4daabf0fa80b4c05142a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3536d21e2a7e4daabf0fa80b4c05142a 2024-11-13T19:24:27,160 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3536d21e2a7e4daabf0fa80b4c05142a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-13T19:24:27,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/fbb11c8412a9408393b89080beb5f73c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/fbb11c8412a9408393b89080beb5f73c 2024-11-13T19:24:27,165 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/fbb11c8412a9408393b89080beb5f73c, entries=150, sequenceid=192, filesize=11.9 K 2024-11-13T19:24:27,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/063f7fb69af34729a7cf000695040359 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/063f7fb69af34729a7cf000695040359 2024-11-13T19:24:27,170 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/063f7fb69af34729a7cf000695040359, entries=150, sequenceid=192, filesize=11.9 K 2024-11-13T19:24:27,171 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b9ac03ed4dfd24d768bc705ca3d577cd in 852ms, sequenceid=192, compaction requested=false 2024-11-13T19:24:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:27,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-13T19:24:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-13T19:24:27,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-13T19:24:27,174 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0060 sec 2024-11-13T19:24:27,177 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0120 sec 2024-11-13T19:24:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-13T19:24:27,301 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:27,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-13T19:24:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:24:27,303 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:27,304 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:27,304 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:24:27,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-13T19:24:27,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:27,456 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T19:24:27,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/0370da2b3fec4d918354e48acc916f42 is 50, key is test_row_0/A:col10/1731525866446/Put/seqid=0 2024-11-13T19:24:27,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742099_1275 (size=12151) 2024-11-13T19:24:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:27,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:24:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525927638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:27,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525927751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:27,867 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/0370da2b3fec4d918354e48acc916f42 2024-11-13T19:24:27,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/adf6183c89ca439fba0be2cf5cf2b60c is 50, key is test_row_0/B:col10/1731525866446/Put/seqid=0 2024-11-13T19:24:27,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742100_1276 (size=12151) 2024-11-13T19:24:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:24:27,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525927961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:27,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:28,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525928271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:28,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:28,278 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/adf6183c89ca439fba0be2cf5cf2b60c 2024-11-13T19:24:28,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/aea64ed420b04cffb57b43d605f01b2b is 50, key is test_row_0/C:col10/1731525866446/Put/seqid=0 2024-11-13T19:24:28,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742101_1277 (size=12151) 2024-11-13T19:24:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:24:28,691 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/aea64ed420b04cffb57b43d605f01b2b 2024-11-13T19:24:28,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/0370da2b3fec4d918354e48acc916f42 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/0370da2b3fec4d918354e48acc916f42 2024-11-13T19:24:28,700 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/0370da2b3fec4d918354e48acc916f42, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T19:24:28,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/adf6183c89ca439fba0be2cf5cf2b60c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/adf6183c89ca439fba0be2cf5cf2b60c 2024-11-13T19:24:28,705 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/adf6183c89ca439fba0be2cf5cf2b60c, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T19:24:28,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/aea64ed420b04cffb57b43d605f01b2b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/aea64ed420b04cffb57b43d605f01b2b 2024-11-13T19:24:28,710 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/aea64ed420b04cffb57b43d605f01b2b, entries=150, sequenceid=213, filesize=11.9 K 2024-11-13T19:24:28,711 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b9ac03ed4dfd24d768bc705ca3d577cd in 1255ms, sequenceid=213, compaction requested=true 2024-11-13T19:24:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:28,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-13T19:24:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-13T19:24:28,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-13T19:24:28,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4080 sec 2024-11-13T19:24:28,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4130 sec 2024-11-13T19:24:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:28,784 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:28,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T19:24:28,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/e377b402b4404aae9fed9f8034e5005e is 50, key is test_row_0/A:col10/1731525867616/Put/seqid=0 2024-11-13T19:24:28,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742102_1278 (size=14541) 2024-11-13T19:24:28,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/e377b402b4404aae9fed9f8034e5005e 2024-11-13T19:24:28,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1aaa28e873dd4b3eb73ec31a1b8eb02c is 50, key is test_row_0/B:col10/1731525867616/Put/seqid=0 2024-11-13T19:24:28,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742103_1279 (size=12151) 2024-11-13T19:24:28,807 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1aaa28e873dd4b3eb73ec31a1b8eb02c 2024-11-13T19:24:28,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/56da885b45524f709c852f37f5e5fa99 is 50, key is test_row_0/C:col10/1731525867616/Put/seqid=0 2024-11-13T19:24:28,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742104_1280 (size=12151) 2024-11-13T19:24:28,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525928866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:28,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:28,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525928982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:28,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525929141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,142 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525929141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525929141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,145 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,145 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525929152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,153 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525929191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/56da885b45524f709c852f37f5e5fa99 2024-11-13T19:24:29,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/e377b402b4404aae9fed9f8034e5005e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/e377b402b4404aae9fed9f8034e5005e 2024-11-13T19:24:29,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/e377b402b4404aae9fed9f8034e5005e, entries=200, sequenceid=230, filesize=14.2 K 2024-11-13T19:24:29,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1aaa28e873dd4b3eb73ec31a1b8eb02c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1aaa28e873dd4b3eb73ec31a1b8eb02c 2024-11-13T19:24:29,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1aaa28e873dd4b3eb73ec31a1b8eb02c, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T19:24:29,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/56da885b45524f709c852f37f5e5fa99 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/56da885b45524f709c852f37f5e5fa99 2024-11-13T19:24:29,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/56da885b45524f709c852f37f5e5fa99, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T19:24:29,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for b9ac03ed4dfd24d768bc705ca3d577cd in 495ms, sequenceid=230, compaction requested=true 2024-11-13T19:24:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:29,279 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:29,279 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:29,279 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:29,280 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:29,280 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51404 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:29,280 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/A is initiating minor compaction (all files) 2024-11-13T19:24:29,280 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/B is initiating minor compaction (all files) 2024-11-13T19:24:29,280 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/B in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:29,280 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/A in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:29,281 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/9b9ec4c799eb4096be40150e0dbd9526, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3536d21e2a7e4daabf0fa80b4c05142a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/0370da2b3fec4d918354e48acc916f42, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/e377b402b4404aae9fed9f8034e5005e] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=50.2 K 2024-11-13T19:24:29,281 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/61b2fcff4cf64a3e8897a6e896b5ba6b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/fbb11c8412a9408393b89080beb5f73c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/adf6183c89ca439fba0be2cf5cf2b60c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1aaa28e873dd4b3eb73ec31a1b8eb02c] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=47.9 K 2024-11-13T19:24:29,281 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b9ec4c799eb4096be40150e0dbd9526, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731525864961 2024-11-13T19:24:29,281 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 61b2fcff4cf64a3e8897a6e896b5ba6b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731525864961 2024-11-13T19:24:29,281 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting fbb11c8412a9408393b89080beb5f73c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731525865736 2024-11-13T19:24:29,281 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3536d21e2a7e4daabf0fa80b4c05142a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731525865736 2024-11-13T19:24:29,282 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting adf6183c89ca439fba0be2cf5cf2b60c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731525866424 2024-11-13T19:24:29,282 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0370da2b3fec4d918354e48acc916f42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731525866424 2024-11-13T19:24:29,282 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aaa28e873dd4b3eb73ec31a1b8eb02c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525867616 2024-11-13T19:24:29,282 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e377b402b4404aae9fed9f8034e5005e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525867616 2024-11-13T19:24:29,290 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:29,290 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#A#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:29,291 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/beb9e546bc0540adb509a4845f17a63c is 50, key is test_row_0/B:col10/1731525867616/Put/seqid=0 2024-11-13T19:24:29,291 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/f5f53835ddb54a1fb6afaabacd7bf271 is 50, key is test_row_0/A:col10/1731525867616/Put/seqid=0 2024-11-13T19:24:29,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742105_1281 (size=12697) 2024-11-13T19:24:29,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742106_1282 (size=12697) 2024-11-13T19:24:29,305 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/beb9e546bc0540adb509a4845f17a63c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/beb9e546bc0540adb509a4845f17a63c 2024-11-13T19:24:29,310 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/B of b9ac03ed4dfd24d768bc705ca3d577cd into beb9e546bc0540adb509a4845f17a63c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:29,310 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:29,310 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/B, priority=12, startTime=1731525869279; duration=0sec 2024-11-13T19:24:29,310 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:29,310 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:B 2024-11-13T19:24:29,310 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:29,312 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:29,312 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/C is initiating minor compaction (all files) 2024-11-13T19:24:29,312 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/C in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:29,312 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8ed87107a93247b6abeeb5dc7a4a8f07, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/063f7fb69af34729a7cf000695040359, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/aea64ed420b04cffb57b43d605f01b2b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/56da885b45524f709c852f37f5e5fa99] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=47.9 K 2024-11-13T19:24:29,313 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ed87107a93247b6abeeb5dc7a4a8f07, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1731525864961 2024-11-13T19:24:29,313 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 063f7fb69af34729a7cf000695040359, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731525865736 2024-11-13T19:24:29,313 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting aea64ed420b04cffb57b43d605f01b2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731525866424 2024-11-13T19:24:29,314 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 56da885b45524f709c852f37f5e5fa99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525867616 2024-11-13T19:24:29,323 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#C#compaction#242 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:29,324 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/385ebaa3bf3f4ebab2a19885662e6a38 is 50, key is test_row_0/C:col10/1731525867616/Put/seqid=0 2024-11-13T19:24:29,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742107_1283 (size=12697) 2024-11-13T19:24:29,334 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/385ebaa3bf3f4ebab2a19885662e6a38 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/385ebaa3bf3f4ebab2a19885662e6a38 2024-11-13T19:24:29,339 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/C of b9ac03ed4dfd24d768bc705ca3d577cd into 385ebaa3bf3f4ebab2a19885662e6a38(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:29,339 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:29,340 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/C, priority=12, startTime=1731525869279; duration=0sec 2024-11-13T19:24:29,340 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:29,340 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:C 2024-11-13T19:24:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-13T19:24:29,452 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:29,452 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-13T19:24:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T19:24:29,454 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:29,455 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:29,455 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:29,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:29,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-13T19:24:29,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/ce2a1f90de8945569287bea207698bb9 is 50, key is test_row_0/A:col10/1731525869504/Put/seqid=0 2024-11-13T19:24:29,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742108_1284 (size=14541) 2024-11-13T19:24:29,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525929558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T19:24:29,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:29,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:29,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:29,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:29,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:29,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:29,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525929671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,674 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:29,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:29,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:29,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:29,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:29,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T19:24:29,868 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/f5f53835ddb54a1fb6afaabacd7bf271 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f5f53835ddb54a1fb6afaabacd7bf271 2024-11-13T19:24:29,872 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/A of b9ac03ed4dfd24d768bc705ca3d577cd into f5f53835ddb54a1fb6afaabacd7bf271(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:29,872 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:29,872 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/A, priority=12, startTime=1731525869279; duration=0sec 2024-11-13T19:24:29,872 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:29,872 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:A 2024-11-13T19:24:29,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:29,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525929881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:29,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:29,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/ce2a1f90de8945569287bea207698bb9 2024-11-13T19:24:29,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/31a034dae71f429abe0a9643094259cd is 50, key is test_row_0/B:col10/1731525869504/Put/seqid=0 2024-11-13T19:24:29,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742109_1285 (size=12151) 2024-11-13T19:24:30,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:30,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:30,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,015 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T19:24:30,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:30,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:30,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525930191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:30,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:30,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:30,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:30,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,324 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/31a034dae71f429abe0a9643094259cd 2024-11-13T19:24:30,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/b5d1b8caee044f80905848b349b876ef is 50, key is test_row_0/C:col10/1731525869504/Put/seqid=0 2024-11-13T19:24:30,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742110_1286 (size=12151) 2024-11-13T19:24:30,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T19:24:30,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:30,703 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525930701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:30,704 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:30,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/b5d1b8caee044f80905848b349b876ef 2024-11-13T19:24:30,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/ce2a1f90de8945569287bea207698bb9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/ce2a1f90de8945569287bea207698bb9 2024-11-13T19:24:30,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/ce2a1f90de8945569287bea207698bb9, entries=200, sequenceid=252, filesize=14.2 K 2024-11-13T19:24:30,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/31a034dae71f429abe0a9643094259cd as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/31a034dae71f429abe0a9643094259cd 2024-11-13T19:24:30,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/31a034dae71f429abe0a9643094259cd, entries=150, sequenceid=252, filesize=11.9 K 2024-11-13T19:24:30,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/b5d1b8caee044f80905848b349b876ef as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/b5d1b8caee044f80905848b349b876ef 2024-11-13T19:24:30,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/b5d1b8caee044f80905848b349b876ef, entries=150, sequenceid=252, filesize=11.9 K 2024-11-13T19:24:30,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b9ac03ed4dfd24d768bc705ca3d577cd in 1252ms, sequenceid=252, compaction requested=false 2024-11-13T19:24:30,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:30,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-13T19:24:30,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,779 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T19:24:30,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/8458d286bd90480387e5678427287716 is 50, key is test_row_0/A:col10/1731525869536/Put/seqid=0 2024-11-13T19:24:30,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742111_1287 (size=12301) 2024-11-13T19:24:30,797 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/8458d286bd90480387e5678427287716 2024-11-13T19:24:30,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/5304063959cb46f093ed1837d1e45a37 is 50, key is test_row_0/B:col10/1731525869536/Put/seqid=0 2024-11-13T19:24:30,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742112_1288 (size=12301) 2024-11-13T19:24:30,807 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/5304063959cb46f093ed1837d1e45a37 2024-11-13T19:24:30,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/2553258d464a497f9af3abc615fdf767 is 50, key is test_row_0/C:col10/1731525869536/Put/seqid=0 2024-11-13T19:24:30,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742113_1289 (size=12301) 2024-11-13T19:24:30,826 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/2553258d464a497f9af3abc615fdf767 2024-11-13T19:24:30,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/8458d286bd90480387e5678427287716 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/8458d286bd90480387e5678427287716 2024-11-13T19:24:30,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/8458d286bd90480387e5678427287716, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T19:24:30,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/5304063959cb46f093ed1837d1e45a37 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5304063959cb46f093ed1837d1e45a37 2024-11-13T19:24:30,844 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5304063959cb46f093ed1837d1e45a37, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T19:24:30,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/2553258d464a497f9af3abc615fdf767 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/2553258d464a497f9af3abc615fdf767 2024-11-13T19:24:30,850 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/2553258d464a497f9af3abc615fdf767, entries=150, sequenceid=269, filesize=12.0 K 2024-11-13T19:24:30,852 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for b9ac03ed4dfd24d768bc705ca3d577cd in 72ms, sequenceid=269, compaction requested=true 2024-11-13T19:24:30,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:30,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:30,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-13T19:24:30,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-13T19:24:30,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-13T19:24:30,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-13T19:24:30,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4050 sec 2024-11-13T19:24:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-13T19:24:31,612 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:31,612 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-13T19:24:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:24:31,614 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:31,615 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:31,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:24:31,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:31,740 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:31,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:24:31,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/87612da95a0a4d17aed56b7cc1736b4b is 50, key is test_row_0/A:col10/1731525871739/Put/seqid=0 2024-11-13T19:24:31,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742114_1290 (size=12301) 2024-11-13T19:24:31,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/87612da95a0a4d17aed56b7cc1736b4b 2024-11-13T19:24:31,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/7aa7af6cffad4bb1868c5108b714b9db is 50, key is test_row_0/B:col10/1731525871739/Put/seqid=0 2024-11-13T19:24:31,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742115_1291 (size=12301) 2024-11-13T19:24:31,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/7aa7af6cffad4bb1868c5108b714b9db 2024-11-13T19:24:31,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/75b6d013f08e43bab9b3f33b02c5684b is 50, key is test_row_0/C:col10/1731525871739/Put/seqid=0 2024-11-13T19:24:31,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:31,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:31,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:31,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:31,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742116_1292 (size=12301) 2024-11-13T19:24:31,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=280 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/75b6d013f08e43bab9b3f33b02c5684b 2024-11-13T19:24:31,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/87612da95a0a4d17aed56b7cc1736b4b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/87612da95a0a4d17aed56b7cc1736b4b 2024-11-13T19:24:31,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/87612da95a0a4d17aed56b7cc1736b4b, entries=150, sequenceid=280, filesize=12.0 K 2024-11-13T19:24:31,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/7aa7af6cffad4bb1868c5108b714b9db as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7aa7af6cffad4bb1868c5108b714b9db 2024-11-13T19:24:31,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7aa7af6cffad4bb1868c5108b714b9db, entries=150, sequenceid=280, filesize=12.0 K 2024-11-13T19:24:31,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/75b6d013f08e43bab9b3f33b02c5684b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/75b6d013f08e43bab9b3f33b02c5684b 2024-11-13T19:24:31,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/75b6d013f08e43bab9b3f33b02c5684b, entries=150, sequenceid=280, filesize=12.0 K 2024-11-13T19:24:31,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=87.22 KB/89310 for b9ac03ed4dfd24d768bc705ca3d577cd in 60ms, sequenceid=280, compaction requested=true 2024-11-13T19:24:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:31,801 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:31,801 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:31,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:31,802 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:31,802 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/B is initiating minor compaction (all files) 2024-11-13T19:24:31,802 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51840 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:31,802 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/A is initiating minor compaction (all files) 2024-11-13T19:24:31,802 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/B in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:31,802 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/A in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:31,802 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f5f53835ddb54a1fb6afaabacd7bf271, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/ce2a1f90de8945569287bea207698bb9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/8458d286bd90480387e5678427287716, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/87612da95a0a4d17aed56b7cc1736b4b] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=50.6 K 2024-11-13T19:24:31,802 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/beb9e546bc0540adb509a4845f17a63c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/31a034dae71f429abe0a9643094259cd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5304063959cb46f093ed1837d1e45a37, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7aa7af6cffad4bb1868c5108b714b9db] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=48.3 K 2024-11-13T19:24:31,803 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5f53835ddb54a1fb6afaabacd7bf271, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525867616 2024-11-13T19:24:31,803 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting beb9e546bc0540adb509a4845f17a63c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525867616 2024-11-13T19:24:31,803 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce2a1f90de8945569287bea207698bb9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525868821 2024-11-13T19:24:31,804 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a034dae71f429abe0a9643094259cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525868821 2024-11-13T19:24:31,804 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8458d286bd90480387e5678427287716, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731525869536 2024-11-13T19:24:31,804 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5304063959cb46f093ed1837d1e45a37, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731525869536 2024-11-13T19:24:31,804 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87612da95a0a4d17aed56b7cc1736b4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731525871734 2024-11-13T19:24:31,804 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7aa7af6cffad4bb1868c5108b714b9db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731525871734 2024-11-13T19:24:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:31,808 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:31,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T19:24:31,812 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:31,813 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#A#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:31,813 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/4fcc2ca1eca54e2bad3ccb18d60db896 is 50, key is test_row_0/B:col10/1731525871739/Put/seqid=0 2024-11-13T19:24:31,813 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/566f85697e494ee99d088563e3b9834a is 50, key is test_row_0/A:col10/1731525871739/Put/seqid=0 2024-11-13T19:24:31,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/4466464e889d4e98a58fd65f49492924 is 50, key is test_row_0/A:col10/1731525871802/Put/seqid=0 2024-11-13T19:24:31,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742117_1293 (size=12983) 2024-11-13T19:24:31,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742118_1294 (size=12983) 2024-11-13T19:24:31,836 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/4fcc2ca1eca54e2bad3ccb18d60db896 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/4fcc2ca1eca54e2bad3ccb18d60db896 2024-11-13T19:24:31,842 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/B of b9ac03ed4dfd24d768bc705ca3d577cd into 4fcc2ca1eca54e2bad3ccb18d60db896(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:31,842 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:31,842 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/B, priority=12, startTime=1731525871801; duration=0sec 2024-11-13T19:24:31,842 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:31,842 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:B 2024-11-13T19:24:31,842 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:31,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742119_1295 (size=14741) 2024-11-13T19:24:31,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/4466464e889d4e98a58fd65f49492924 2024-11-13T19:24:31,845 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:31,845 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/C is initiating minor compaction (all files) 2024-11-13T19:24:31,845 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/C in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:31,845 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/385ebaa3bf3f4ebab2a19885662e6a38, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/b5d1b8caee044f80905848b349b876ef, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/2553258d464a497f9af3abc615fdf767, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/75b6d013f08e43bab9b3f33b02c5684b] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=48.3 K 2024-11-13T19:24:31,845 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 385ebaa3bf3f4ebab2a19885662e6a38, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525867616 2024-11-13T19:24:31,846 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting b5d1b8caee044f80905848b349b876ef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731525868821 2024-11-13T19:24:31,846 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2553258d464a497f9af3abc615fdf767, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1731525869536 2024-11-13T19:24:31,846 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b6d013f08e43bab9b3f33b02c5684b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731525871734 2024-11-13T19:24:31,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/e2902101d98441e3971aa5895fbe02eb is 50, key is test_row_0/B:col10/1731525871802/Put/seqid=0 2024-11-13T19:24:31,856 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#C#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:31,857 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/9b2c610ef1344dc39dbfdec81c047930 is 50, key is test_row_0/C:col10/1731525871739/Put/seqid=0 2024-11-13T19:24:31,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742120_1296 (size=12301) 2024-11-13T19:24:31,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742121_1297 (size=12983) 2024-11-13T19:24:31,869 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/9b2c610ef1344dc39dbfdec81c047930 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/9b2c610ef1344dc39dbfdec81c047930 2024-11-13T19:24:31,873 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/C of b9ac03ed4dfd24d768bc705ca3d577cd into 9b2c610ef1344dc39dbfdec81c047930(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:31,873 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:31,873 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/C, priority=12, startTime=1731525871801; duration=0sec 2024-11-13T19:24:31,873 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:31,873 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:C 2024-11-13T19:24:31,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525931893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:31,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:31,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:31,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:31,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:24:32,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525932001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:32,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:32,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525932211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:32,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:32,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,234 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/566f85697e494ee99d088563e3b9834a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/566f85697e494ee99d088563e3b9834a 2024-11-13T19:24:32,239 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/A of b9ac03ed4dfd24d768bc705ca3d577cd into 566f85697e494ee99d088563e3b9834a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:32,239 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:32,239 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/A, priority=12, startTime=1731525871801; duration=0sec 2024-11-13T19:24:32,239 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:32,239 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:A 2024-11-13T19:24:32,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:24:32,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/e2902101d98441e3971aa5895fbe02eb 2024-11-13T19:24:32,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/c1ceaf41ce2349608d4d6b414f04d64e is 50, key is test_row_0/C:col10/1731525871802/Put/seqid=0 2024-11-13T19:24:32,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742122_1298 (size=12301) 2024-11-13T19:24:32,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:32,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525932521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:32,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:32,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/c1ceaf41ce2349608d4d6b414f04d64e 2024-11-13T19:24:32,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/4466464e889d4e98a58fd65f49492924 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4466464e889d4e98a58fd65f49492924 2024-11-13T19:24:32,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4466464e889d4e98a58fd65f49492924, entries=200, sequenceid=298, filesize=14.4 K 2024-11-13T19:24:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/e2902101d98441e3971aa5895fbe02eb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/e2902101d98441e3971aa5895fbe02eb 2024-11-13T19:24:32,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:32,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/e2902101d98441e3971aa5895fbe02eb, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T19:24:32,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/c1ceaf41ce2349608d4d6b414f04d64e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c1ceaf41ce2349608d4d6b414f04d64e 2024-11-13T19:24:32,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c1ceaf41ce2349608d4d6b414f04d64e, entries=150, sequenceid=298, filesize=12.0 K 2024-11-13T19:24:32,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b9ac03ed4dfd24d768bc705ca3d577cd in 887ms, sequenceid=298, compaction requested=false 2024-11-13T19:24:32,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:32,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:24:32,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-13T19:24:32,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:32,838 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T19:24:32,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6e3810957b50451280b55f06c56b06a6 is 50, key is test_row_0/A:col10/1731525871891/Put/seqid=0 2024-11-13T19:24:32,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742123_1299 (size=12301) 2024-11-13T19:24:32,848 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6e3810957b50451280b55f06c56b06a6 2024-11-13T19:24:32,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/72c571aeb3b647e28c6e34a2e2238ad4 is 50, key is test_row_0/B:col10/1731525871891/Put/seqid=0 2024-11-13T19:24:32,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742124_1300 (size=12301) 2024-11-13T19:24:33,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:33,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:33,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525933100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41094 deadline: 1731525933161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,165 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,166 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41102 deadline: 1731525933171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,174 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41138 deadline: 1731525933172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,175 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41152 deadline: 1731525933181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,184 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b9ac03ed4dfd24d768bc705ca3d577cd of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525933211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,260 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/72c571aeb3b647e28c6e34a2e2238ad4 2024-11-13T19:24:33,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/375011b0496248e480fec62bf3840cfb is 50, key is test_row_0/C:col10/1731525871891/Put/seqid=0 2024-11-13T19:24:33,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742125_1301 (size=12301) 2024-11-13T19:24:33,271 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/375011b0496248e480fec62bf3840cfb 2024-11-13T19:24:33,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6e3810957b50451280b55f06c56b06a6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e3810957b50451280b55f06c56b06a6 2024-11-13T19:24:33,284 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e3810957b50451280b55f06c56b06a6, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T19:24:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/72c571aeb3b647e28c6e34a2e2238ad4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/72c571aeb3b647e28c6e34a2e2238ad4 2024-11-13T19:24:33,288 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/72c571aeb3b647e28c6e34a2e2238ad4, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T19:24:33,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/375011b0496248e480fec62bf3840cfb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/375011b0496248e480fec62bf3840cfb 2024-11-13T19:24:33,293 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/375011b0496248e480fec62bf3840cfb, entries=150, sequenceid=319, filesize=12.0 K 2024-11-13T19:24:33,293 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for b9ac03ed4dfd24d768bc705ca3d577cd in 455ms, sequenceid=319, compaction requested=true 2024-11-13T19:24:33,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:33,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:33,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-13T19:24:33,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-13T19:24:33,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-13T19:24:33,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6800 sec 2024-11-13T19:24:33,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.6850 sec 2024-11-13T19:24:33,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:33,423 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:33,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T19:24:33,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6efed1e1f01e40a9bd474d7d0cf3000b is 50, key is test_row_0/A:col10/1731525873422/Put/seqid=0 2024-11-13T19:24:33,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742126_1302 (size=12301) 2024-11-13T19:24:33,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525933493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525933601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,603 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-13T19:24:33,762 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:33,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:33,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-13T19:24:33,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T19:24:33,764 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:33,765 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:33,765 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:33,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:33,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525933811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:33,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:33,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6efed1e1f01e40a9bd474d7d0cf3000b 2024-11-13T19:24:33,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/f8ff150d7ad0497b900262e047020fcd is 50, key is test_row_0/B:col10/1731525873422/Put/seqid=0 2024-11-13T19:24:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T19:24:33,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742127_1303 (size=12301) 2024-11-13T19:24:33,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T19:24:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:33,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:33,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:33,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T19:24:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T19:24:34,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:34,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525934122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:34,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:34,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T19:24:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/f8ff150d7ad0497b900262e047020fcd 2024-11-13T19:24:34,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/8bcedbb81e2d457cbd58aba2170de06b is 50, key is test_row_0/C:col10/1731525873422/Put/seqid=0 2024-11-13T19:24:34,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742128_1304 (size=12301) 2024-11-13T19:24:34,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T19:24:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T19:24:34,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T19:24:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:34,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41116 deadline: 1731525934631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:34,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:24:34,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T19:24:34,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:34,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:34,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/8bcedbb81e2d457cbd58aba2170de06b 2024-11-13T19:24:34,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/6efed1e1f01e40a9bd474d7d0cf3000b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6efed1e1f01e40a9bd474d7d0cf3000b 2024-11-13T19:24:34,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6efed1e1f01e40a9bd474d7d0cf3000b, entries=150, sequenceid=338, filesize=12.0 K 2024-11-13T19:24:34,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/f8ff150d7ad0497b900262e047020fcd as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/f8ff150d7ad0497b900262e047020fcd 2024-11-13T19:24:34,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/f8ff150d7ad0497b900262e047020fcd, entries=150, sequenceid=338, filesize=12.0 K 2024-11-13T19:24:34,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/8bcedbb81e2d457cbd58aba2170de06b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8bcedbb81e2d457cbd58aba2170de06b 2024-11-13T19:24:34,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8bcedbb81e2d457cbd58aba2170de06b, entries=150, sequenceid=338, filesize=12.0 K 2024-11-13T19:24:34,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for b9ac03ed4dfd24d768bc705ca3d577cd in 1295ms, sequenceid=338, compaction requested=true 2024-11-13T19:24:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:34,719 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:34,719 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b9ac03ed4dfd24d768bc705ca3d577cd:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:34,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:34,720 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52326 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:34,720 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:34,720 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/B is initiating minor compaction (all files) 2024-11-13T19:24:34,720 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/A is initiating minor compaction (all files) 2024-11-13T19:24:34,720 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/A in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,720 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/B in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,720 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/566f85697e494ee99d088563e3b9834a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4466464e889d4e98a58fd65f49492924, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e3810957b50451280b55f06c56b06a6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6efed1e1f01e40a9bd474d7d0cf3000b] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=51.1 K 2024-11-13T19:24:34,720 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/4fcc2ca1eca54e2bad3ccb18d60db896, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/e2902101d98441e3971aa5895fbe02eb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/72c571aeb3b647e28c6e34a2e2238ad4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/f8ff150d7ad0497b900262e047020fcd] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=48.7 K 2024-11-13T19:24:34,721 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcc2ca1eca54e2bad3ccb18d60db896, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731525871734 2024-11-13T19:24:34,721 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 566f85697e494ee99d088563e3b9834a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731525871734 2024-11-13T19:24:34,721 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e2902101d98441e3971aa5895fbe02eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731525871781 2024-11-13T19:24:34,721 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4466464e889d4e98a58fd65f49492924, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731525871781 2024-11-13T19:24:34,721 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 72c571aeb3b647e28c6e34a2e2238ad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731525871871 2024-11-13T19:24:34,721 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e3810957b50451280b55f06c56b06a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731525871871 2024-11-13T19:24:34,722 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f8ff150d7ad0497b900262e047020fcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731525873088 2024-11-13T19:24:34,722 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6efed1e1f01e40a9bd474d7d0cf3000b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731525873088 2024-11-13T19:24:34,730 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#B#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:34,730 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#A#compaction#265 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:34,730 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1996e11158cd461482f0a68737f2d916 is 50, key is test_row_0/B:col10/1731525873422/Put/seqid=0 2024-11-13T19:24:34,731 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/38fb4925441d4e78bca12b38a7680a43 is 50, key is test_row_0/A:col10/1731525873422/Put/seqid=0 2024-11-13T19:24:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742129_1305 (size=13119) 2024-11-13T19:24:34,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742130_1306 (size=13119) 2024-11-13T19:24:34,742 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/1996e11158cd461482f0a68737f2d916 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1996e11158cd461482f0a68737f2d916 2024-11-13T19:24:34,748 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/B of b9ac03ed4dfd24d768bc705ca3d577cd into 1996e11158cd461482f0a68737f2d916(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:34,748 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:34,748 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/B, priority=12, startTime=1731525874719; duration=0sec 2024-11-13T19:24:34,748 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:34,749 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:B 2024-11-13T19:24:34,749 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:34,750 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:34,750 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b9ac03ed4dfd24d768bc705ca3d577cd/C is initiating minor compaction (all files) 2024-11-13T19:24:34,750 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b9ac03ed4dfd24d768bc705ca3d577cd/C in TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,750 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/9b2c610ef1344dc39dbfdec81c047930, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c1ceaf41ce2349608d4d6b414f04d64e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/375011b0496248e480fec62bf3840cfb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8bcedbb81e2d457cbd58aba2170de06b] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp, totalSize=48.7 K 2024-11-13T19:24:34,750 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b2c610ef1344dc39dbfdec81c047930, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=280, earliestPutTs=1731525871734 2024-11-13T19:24:34,751 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ceaf41ce2349608d4d6b414f04d64e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=298, earliestPutTs=1731525871781 2024-11-13T19:24:34,751 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 375011b0496248e480fec62bf3840cfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1731525871871 2024-11-13T19:24:34,751 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bcedbb81e2d457cbd58aba2170de06b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1731525873088 2024-11-13T19:24:34,757 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b9ac03ed4dfd24d768bc705ca3d577cd#C#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:34,758 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/68ec1d793d194622b8bdef3058d4c469 is 50, key is test_row_0/C:col10/1731525873422/Put/seqid=0 2024-11-13T19:24:34,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742131_1307 (size=13119) 2024-11-13T19:24:34,765 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/68ec1d793d194622b8bdef3058d4c469 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/68ec1d793d194622b8bdef3058d4c469 2024-11-13T19:24:34,769 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/C of b9ac03ed4dfd24d768bc705ca3d577cd into 68ec1d793d194622b8bdef3058d4c469(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:34,769 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:34,769 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/C, priority=12, startTime=1731525874719; duration=0sec 2024-11-13T19:24:34,769 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:34,769 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:C 2024-11-13T19:24:34,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-13T19:24:34,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:34,834 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T19:24:34,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/f48b2509ce12437b88d935f8819b3f88 is 50, key is test_row_0/A:col10/1731525873491/Put/seqid=0 2024-11-13T19:24:34,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742132_1308 (size=12301) 2024-11-13T19:24:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T19:24:35,141 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/38fb4925441d4e78bca12b38a7680a43 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/38fb4925441d4e78bca12b38a7680a43 2024-11-13T19:24:35,145 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b9ac03ed4dfd24d768bc705ca3d577cd/A of b9ac03ed4dfd24d768bc705ca3d577cd into 38fb4925441d4e78bca12b38a7680a43(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:35,145 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:35,145 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., storeName=b9ac03ed4dfd24d768bc705ca3d577cd/A, priority=12, startTime=1731525874719; duration=0sec 2024-11-13T19:24:35,145 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:35,145 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b9ac03ed4dfd24d768bc705ca3d577cd:A 2024-11-13T19:24:35,241 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/f48b2509ce12437b88d935f8819b3f88 2024-11-13T19:24:35,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/c2d505584898496e889fee7ccffe3b3a is 50, key is test_row_0/B:col10/1731525873491/Put/seqid=0 2024-11-13T19:24:35,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742133_1309 (size=12301) 2024-11-13T19:24:35,524 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-13T19:24:35,524 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-13T19:24:35,524 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:35,524 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:35,524 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,524 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:35,524 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,524 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:35,524 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,524 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-13T19:24:35,524 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:35,524 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,525 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:35,526 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-13T19:24:35,526 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-13T19:24:35,526 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:35,526 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:35,526 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,526 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,526 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:35,526 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:35,526 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:35,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. as already flushing 2024-11-13T19:24:35,647 INFO [Thread-1090 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1090. 2024-11-13T19:24:35,647 DEBUG [Thread-1090 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:35,647 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,648 DEBUG [Thread-1090 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:35,648 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:35,654 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/c2d505584898496e889fee7ccffe3b3a 2024-11-13T19:24:35,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/cae7544bee354814bfa7f6c4c279387e is 50, key is test_row_0/C:col10/1731525873491/Put/seqid=0 2024-11-13T19:24:35,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742134_1310 (size=12301) 2024-11-13T19:24:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T19:24:36,071 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/cae7544bee354814bfa7f6c4c279387e 2024-11-13T19:24:36,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/f48b2509ce12437b88d935f8819b3f88 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f48b2509ce12437b88d935f8819b3f88 2024-11-13T19:24:36,088 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f48b2509ce12437b88d935f8819b3f88, entries=150, sequenceid=357, filesize=12.0 K 2024-11-13T19:24:36,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/c2d505584898496e889fee7ccffe3b3a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2d505584898496e889fee7ccffe3b3a 2024-11-13T19:24:36,095 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2d505584898496e889fee7ccffe3b3a, entries=150, sequenceid=357, filesize=12.0 K 2024-11-13T19:24:36,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/cae7544bee354814bfa7f6c4c279387e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/cae7544bee354814bfa7f6c4c279387e 2024-11-13T19:24:36,101 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/cae7544bee354814bfa7f6c4c279387e, entries=150, sequenceid=357, filesize=12.0 K 2024-11-13T19:24:36,101 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=6.71 KB/6870 for b9ac03ed4dfd24d768bc705ca3d577cd in 1267ms, sequenceid=357, compaction requested=false 2024-11-13T19:24:36,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for b9ac03ed4dfd24d768bc705ca3d577cd: 2024-11-13T19:24:36,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:36,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-13T19:24:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-13T19:24:36,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-13T19:24:36,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-13T19:24:36,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.3430 sec 2024-11-13T19:24:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-13T19:24:37,943 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:40,752 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:24:43,226 INFO [Thread-1094 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1094. 2024-11-13T19:24:43,226 DEBUG [Thread-1094 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:43,227 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,227 DEBUG [Thread-1094 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,227 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:43,246 INFO [Thread-1086 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1086. 2024-11-13T19:24:43,246 DEBUG [Thread-1086 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:43,246 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,246 DEBUG [Thread-1086 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:43,285 INFO [Thread-1088 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1088. 2024-11-13T19:24:43,285 INFO [Thread-1092 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1092. 2024-11-13T19:24:43,286 DEBUG [Thread-1088 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:43,286 DEBUG [Thread-1092 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:24:43,286 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,286 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,286 DEBUG [Thread-1088 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,286 DEBUG [Thread-1092 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:43,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 191 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4178 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4060 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4033 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4165 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4058 2024-11-13T19:24:43,287 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:24:43,287 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:24:43,287 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:24:43,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,288 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:43,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:24:43,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:24:43,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T19:24:43,293 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525883293"}]},"ts":"1731525883293"} 2024-11-13T19:24:43,295 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:24:43,295 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:24:43,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:24:43,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, UNASSIGN}] 2024-11-13T19:24:43,298 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, UNASSIGN 2024-11-13T19:24:43,299 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b9ac03ed4dfd24d768bc705ca3d577cd, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:43,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, UNASSIGN because future has completed 2024-11-13T19:24:43,301 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:24:43,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:24:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T19:24:43,455 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:43,455 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:24:43,455 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing b9ac03ed4dfd24d768bc705ca3d577cd, disabling compactions & flushes 2024-11-13T19:24:43,455 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:43,455 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:43,455 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. after waiting 0 ms 2024-11-13T19:24:43,455 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:43,456 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing b9ac03ed4dfd24d768bc705ca3d577cd 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T19:24:43,464 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/7330c7376be8480284d99250f6a99acc is 50, key is test_row_0/A:col10/1731525883283/Put/seqid=0 2024-11-13T19:24:43,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742135_1311 (size=12301) 2024-11-13T19:24:43,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T19:24:43,871 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/7330c7376be8480284d99250f6a99acc 2024-11-13T19:24:43,883 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/c2ed3f894ea74b7bb4eea71c0f0d3b27 is 50, key is test_row_0/B:col10/1731525883283/Put/seqid=0 2024-11-13T19:24:43,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742136_1312 (size=12301) 2024-11-13T19:24:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T19:24:44,288 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/c2ed3f894ea74b7bb4eea71c0f0d3b27 2024-11-13T19:24:44,302 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/1064f2609273405e9697ef8e67480b20 is 50, key is test_row_0/C:col10/1731525883283/Put/seqid=0 2024-11-13T19:24:44,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742137_1313 (size=12301) 2024-11-13T19:24:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T19:24:44,708 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/1064f2609273405e9697ef8e67480b20 2024-11-13T19:24:44,719 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/A/7330c7376be8480284d99250f6a99acc as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/7330c7376be8480284d99250f6a99acc 2024-11-13T19:24:44,722 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/7330c7376be8480284d99250f6a99acc, entries=150, sequenceid=366, filesize=12.0 K 2024-11-13T19:24:44,723 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/B/c2ed3f894ea74b7bb4eea71c0f0d3b27 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2ed3f894ea74b7bb4eea71c0f0d3b27 2024-11-13T19:24:44,727 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2ed3f894ea74b7bb4eea71c0f0d3b27, entries=150, sequenceid=366, filesize=12.0 K 2024-11-13T19:24:44,727 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/.tmp/C/1064f2609273405e9697ef8e67480b20 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/1064f2609273405e9697ef8e67480b20 2024-11-13T19:24:44,731 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/1064f2609273405e9697ef8e67480b20, entries=150, sequenceid=366, filesize=12.0 K 2024-11-13T19:24:44,731 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b9ac03ed4dfd24d768bc705ca3d577cd in 1276ms, sequenceid=366, compaction requested=true 2024-11-13T19:24:44,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/94400ab5df65412c9b4d9090d04dd7e7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3b7589ac571848d08902281e2cb3f3db, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/d6773754a334412e9f90a251a2370cde, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/2e6767a0212342c28ee13ece095d4e54, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/49c42cb5d1c143ecbb7f803979b8c405, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/90e5e4d26ad449a29cfda8e3cd3e9da4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/583e527368344d33b67aebb5f17d3359, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6a0f672faab74325b1630d42fad222be, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/a58196d268ba4d1d9b90b0c4be01131d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4388f545e9fc4fe0977c2e7d8cc88f98, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e5286a9c5824960942c088e1dc9cdbf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6df411c531ad449fb596c46bae0d2c92, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/9b9ec4c799eb4096be40150e0dbd9526, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3536d21e2a7e4daabf0fa80b4c05142a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/0370da2b3fec4d918354e48acc916f42, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/e377b402b4404aae9fed9f8034e5005e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f5f53835ddb54a1fb6afaabacd7bf271, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/ce2a1f90de8945569287bea207698bb9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/8458d286bd90480387e5678427287716, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/566f85697e494ee99d088563e3b9834a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/87612da95a0a4d17aed56b7cc1736b4b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4466464e889d4e98a58fd65f49492924, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e3810957b50451280b55f06c56b06a6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6efed1e1f01e40a9bd474d7d0cf3000b] to archive 2024-11-13T19:24:44,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:24:44,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/94400ab5df65412c9b4d9090d04dd7e7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/94400ab5df65412c9b4d9090d04dd7e7 2024-11-13T19:24:44,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3b7589ac571848d08902281e2cb3f3db to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3b7589ac571848d08902281e2cb3f3db 2024-11-13T19:24:44,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/d6773754a334412e9f90a251a2370cde to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/d6773754a334412e9f90a251a2370cde 2024-11-13T19:24:44,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/2e6767a0212342c28ee13ece095d4e54 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/2e6767a0212342c28ee13ece095d4e54 2024-11-13T19:24:44,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/49c42cb5d1c143ecbb7f803979b8c405 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/49c42cb5d1c143ecbb7f803979b8c405 2024-11-13T19:24:44,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/90e5e4d26ad449a29cfda8e3cd3e9da4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/90e5e4d26ad449a29cfda8e3cd3e9da4 2024-11-13T19:24:44,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/583e527368344d33b67aebb5f17d3359 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/583e527368344d33b67aebb5f17d3359 2024-11-13T19:24:44,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6a0f672faab74325b1630d42fad222be to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6a0f672faab74325b1630d42fad222be 2024-11-13T19:24:44,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/a58196d268ba4d1d9b90b0c4be01131d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/a58196d268ba4d1d9b90b0c4be01131d 2024-11-13T19:24:44,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4388f545e9fc4fe0977c2e7d8cc88f98 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4388f545e9fc4fe0977c2e7d8cc88f98 2024-11-13T19:24:44,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e5286a9c5824960942c088e1dc9cdbf to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e5286a9c5824960942c088e1dc9cdbf 2024-11-13T19:24:44,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6df411c531ad449fb596c46bae0d2c92 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6df411c531ad449fb596c46bae0d2c92 2024-11-13T19:24:44,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/9b9ec4c799eb4096be40150e0dbd9526 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/9b9ec4c799eb4096be40150e0dbd9526 2024-11-13T19:24:44,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3536d21e2a7e4daabf0fa80b4c05142a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/3536d21e2a7e4daabf0fa80b4c05142a 2024-11-13T19:24:44,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/0370da2b3fec4d918354e48acc916f42 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/0370da2b3fec4d918354e48acc916f42 2024-11-13T19:24:44,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/e377b402b4404aae9fed9f8034e5005e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/e377b402b4404aae9fed9f8034e5005e 2024-11-13T19:24:44,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f5f53835ddb54a1fb6afaabacd7bf271 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f5f53835ddb54a1fb6afaabacd7bf271 2024-11-13T19:24:44,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/ce2a1f90de8945569287bea207698bb9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/ce2a1f90de8945569287bea207698bb9 2024-11-13T19:24:44,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/8458d286bd90480387e5678427287716 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/8458d286bd90480387e5678427287716 2024-11-13T19:24:44,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/566f85697e494ee99d088563e3b9834a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/566f85697e494ee99d088563e3b9834a 2024-11-13T19:24:44,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/87612da95a0a4d17aed56b7cc1736b4b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/87612da95a0a4d17aed56b7cc1736b4b 2024-11-13T19:24:44,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4466464e889d4e98a58fd65f49492924 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/4466464e889d4e98a58fd65f49492924 2024-11-13T19:24:44,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e3810957b50451280b55f06c56b06a6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6e3810957b50451280b55f06c56b06a6 2024-11-13T19:24:44,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6efed1e1f01e40a9bd474d7d0cf3000b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/6efed1e1f01e40a9bd474d7d0cf3000b 2024-11-13T19:24:44,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0d28c8832ff945dca76bc1001eb24cef, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1f3503e89f7a4ab9a932b4ff12bed8ce, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7ed13f62b690481caf54b631e47e7136, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/054e0463d388439cb6f586fc0b87169d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0e9c20d1698545379e47431f10837655, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/03e2dcf61b634ca38a8be440a3a0734b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5fea97962ace476aa84d7759032a3fec, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3a8e6c272f4746f9a5bd8dbba1873f05, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/01cebd83b5ca4241b0bcd11c791b95a4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/69061f33195542ed960c3a9f8b946fb8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/61b2fcff4cf64a3e8897a6e896b5ba6b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/d2a8c87fed314b159ca0c4480612f75e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/fbb11c8412a9408393b89080beb5f73c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/adf6183c89ca439fba0be2cf5cf2b60c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/beb9e546bc0540adb509a4845f17a63c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1aaa28e873dd4b3eb73ec31a1b8eb02c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/31a034dae71f429abe0a9643094259cd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5304063959cb46f093ed1837d1e45a37, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/4fcc2ca1eca54e2bad3ccb18d60db896, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7aa7af6cffad4bb1868c5108b714b9db, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/e2902101d98441e3971aa5895fbe02eb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/72c571aeb3b647e28c6e34a2e2238ad4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/f8ff150d7ad0497b900262e047020fcd] to archive 2024-11-13T19:24:44,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:24:44,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0d28c8832ff945dca76bc1001eb24cef to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0d28c8832ff945dca76bc1001eb24cef 2024-11-13T19:24:44,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1f3503e89f7a4ab9a932b4ff12bed8ce to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1f3503e89f7a4ab9a932b4ff12bed8ce 2024-11-13T19:24:44,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7ed13f62b690481caf54b631e47e7136 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7ed13f62b690481caf54b631e47e7136 2024-11-13T19:24:44,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/054e0463d388439cb6f586fc0b87169d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/054e0463d388439cb6f586fc0b87169d 2024-11-13T19:24:44,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0e9c20d1698545379e47431f10837655 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/0e9c20d1698545379e47431f10837655 2024-11-13T19:24:44,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/03e2dcf61b634ca38a8be440a3a0734b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/03e2dcf61b634ca38a8be440a3a0734b 2024-11-13T19:24:44,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5fea97962ace476aa84d7759032a3fec to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5fea97962ace476aa84d7759032a3fec 2024-11-13T19:24:44,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3a8e6c272f4746f9a5bd8dbba1873f05 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3a8e6c272f4746f9a5bd8dbba1873f05 2024-11-13T19:24:44,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/01cebd83b5ca4241b0bcd11c791b95a4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/01cebd83b5ca4241b0bcd11c791b95a4 2024-11-13T19:24:44,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/69061f33195542ed960c3a9f8b946fb8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/69061f33195542ed960c3a9f8b946fb8 2024-11-13T19:24:44,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/3bb2ebd4c7bc49fdb5e5dbd2b3dbe083 2024-11-13T19:24:44,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/61b2fcff4cf64a3e8897a6e896b5ba6b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/61b2fcff4cf64a3e8897a6e896b5ba6b 2024-11-13T19:24:44,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/d2a8c87fed314b159ca0c4480612f75e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/d2a8c87fed314b159ca0c4480612f75e 2024-11-13T19:24:44,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/fbb11c8412a9408393b89080beb5f73c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/fbb11c8412a9408393b89080beb5f73c 2024-11-13T19:24:44,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/adf6183c89ca439fba0be2cf5cf2b60c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/adf6183c89ca439fba0be2cf5cf2b60c 2024-11-13T19:24:44,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/beb9e546bc0540adb509a4845f17a63c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/beb9e546bc0540adb509a4845f17a63c 2024-11-13T19:24:44,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1aaa28e873dd4b3eb73ec31a1b8eb02c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1aaa28e873dd4b3eb73ec31a1b8eb02c 2024-11-13T19:24:44,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/31a034dae71f429abe0a9643094259cd to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/31a034dae71f429abe0a9643094259cd 2024-11-13T19:24:44,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5304063959cb46f093ed1837d1e45a37 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/5304063959cb46f093ed1837d1e45a37 2024-11-13T19:24:44,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/4fcc2ca1eca54e2bad3ccb18d60db896 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/4fcc2ca1eca54e2bad3ccb18d60db896 2024-11-13T19:24:44,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7aa7af6cffad4bb1868c5108b714b9db to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/7aa7af6cffad4bb1868c5108b714b9db 2024-11-13T19:24:44,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/e2902101d98441e3971aa5895fbe02eb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/e2902101d98441e3971aa5895fbe02eb 2024-11-13T19:24:44,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/72c571aeb3b647e28c6e34a2e2238ad4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/72c571aeb3b647e28c6e34a2e2238ad4 2024-11-13T19:24:44,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/f8ff150d7ad0497b900262e047020fcd to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/f8ff150d7ad0497b900262e047020fcd 2024-11-13T19:24:44,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/acae3fc3d54549d780aee6544ff56269, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/702637a31c054d5ea595e5711d9336a2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/6474133ea7ba45d6bb94ec4f7e206f9f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c6500d1798e74717af0dff93472aecec, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/261739c59234453b8a3a592367a2bea8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/59d12492a92949f386426163ef789ea7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/480c0d6655d24fb898ef1ae333b306f5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/0481e62875af43b5a28b693bc27f8d46, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/d1ff1302faca4bcfb4aa1692903e7f58, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/fe2a1b1535ae41dbaab6d8628e564cd5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/bc7c51df5d1642d7a33afbe42a1815e0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8ed87107a93247b6abeeb5dc7a4a8f07, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/068b64683bf74447aa5f788537897441, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/063f7fb69af34729a7cf000695040359, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/aea64ed420b04cffb57b43d605f01b2b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/385ebaa3bf3f4ebab2a19885662e6a38, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/56da885b45524f709c852f37f5e5fa99, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/b5d1b8caee044f80905848b349b876ef, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/2553258d464a497f9af3abc615fdf767, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/9b2c610ef1344dc39dbfdec81c047930, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/75b6d013f08e43bab9b3f33b02c5684b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c1ceaf41ce2349608d4d6b414f04d64e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/375011b0496248e480fec62bf3840cfb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8bcedbb81e2d457cbd58aba2170de06b] to archive 2024-11-13T19:24:44,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:24:44,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/acae3fc3d54549d780aee6544ff56269 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/acae3fc3d54549d780aee6544ff56269 2024-11-13T19:24:44,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/702637a31c054d5ea595e5711d9336a2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/702637a31c054d5ea595e5711d9336a2 2024-11-13T19:24:44,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/6474133ea7ba45d6bb94ec4f7e206f9f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/6474133ea7ba45d6bb94ec4f7e206f9f 2024-11-13T19:24:44,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c6500d1798e74717af0dff93472aecec to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c6500d1798e74717af0dff93472aecec 2024-11-13T19:24:44,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/261739c59234453b8a3a592367a2bea8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/261739c59234453b8a3a592367a2bea8 2024-11-13T19:24:44,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/59d12492a92949f386426163ef789ea7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/59d12492a92949f386426163ef789ea7 2024-11-13T19:24:44,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/480c0d6655d24fb898ef1ae333b306f5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/480c0d6655d24fb898ef1ae333b306f5 2024-11-13T19:24:44,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/0481e62875af43b5a28b693bc27f8d46 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/0481e62875af43b5a28b693bc27f8d46 2024-11-13T19:24:44,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/d1ff1302faca4bcfb4aa1692903e7f58 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/d1ff1302faca4bcfb4aa1692903e7f58 2024-11-13T19:24:44,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/fe2a1b1535ae41dbaab6d8628e564cd5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/fe2a1b1535ae41dbaab6d8628e564cd5 2024-11-13T19:24:44,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/bc7c51df5d1642d7a33afbe42a1815e0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/bc7c51df5d1642d7a33afbe42a1815e0 2024-11-13T19:24:44,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8ed87107a93247b6abeeb5dc7a4a8f07 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8ed87107a93247b6abeeb5dc7a4a8f07 2024-11-13T19:24:44,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/068b64683bf74447aa5f788537897441 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/068b64683bf74447aa5f788537897441 2024-11-13T19:24:44,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/063f7fb69af34729a7cf000695040359 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/063f7fb69af34729a7cf000695040359 2024-11-13T19:24:44,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/aea64ed420b04cffb57b43d605f01b2b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/aea64ed420b04cffb57b43d605f01b2b 2024-11-13T19:24:44,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/385ebaa3bf3f4ebab2a19885662e6a38 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/385ebaa3bf3f4ebab2a19885662e6a38 2024-11-13T19:24:44,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/56da885b45524f709c852f37f5e5fa99 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/56da885b45524f709c852f37f5e5fa99 2024-11-13T19:24:44,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/b5d1b8caee044f80905848b349b876ef to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/b5d1b8caee044f80905848b349b876ef 2024-11-13T19:24:44,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/2553258d464a497f9af3abc615fdf767 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/2553258d464a497f9af3abc615fdf767 2024-11-13T19:24:44,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/9b2c610ef1344dc39dbfdec81c047930 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/9b2c610ef1344dc39dbfdec81c047930 2024-11-13T19:24:44,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/75b6d013f08e43bab9b3f33b02c5684b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/75b6d013f08e43bab9b3f33b02c5684b 2024-11-13T19:24:44,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c1ceaf41ce2349608d4d6b414f04d64e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/c1ceaf41ce2349608d4d6b414f04d64e 2024-11-13T19:24:44,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/375011b0496248e480fec62bf3840cfb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/375011b0496248e480fec62bf3840cfb 2024-11-13T19:24:44,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8bcedbb81e2d457cbd58aba2170de06b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/8bcedbb81e2d457cbd58aba2170de06b 2024-11-13T19:24:44,803 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/recovered.edits/369.seqid, newMaxSeqId=369, maxSeqId=1 2024-11-13T19:24:44,804 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. 2024-11-13T19:24:44,804 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for b9ac03ed4dfd24d768bc705ca3d577cd: Waiting for close lock at 1731525883455Running coprocessor pre-close hooks at 1731525883455Disabling compacts and flushes for region at 1731525883455Disabling writes for close at 1731525883455Obtaining lock to block concurrent updates at 1731525883456 (+1 ms)Preparing flush snapshotting stores in b9ac03ed4dfd24d768bc705ca3d577cd at 1731525883456Finished memstore snapshotting TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731525883457 (+1 ms)Flushing stores of TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd. at 1731525883458 (+1 ms)Flushing b9ac03ed4dfd24d768bc705ca3d577cd/A: creating writer at 1731525883458Flushing b9ac03ed4dfd24d768bc705ca3d577cd/A: appending metadata at 1731525883464 (+6 ms)Flushing b9ac03ed4dfd24d768bc705ca3d577cd/A: closing flushed file at 1731525883464Flushing b9ac03ed4dfd24d768bc705ca3d577cd/B: creating writer at 1731525883880 (+416 ms)Flushing b9ac03ed4dfd24d768bc705ca3d577cd/B: appending metadata at 1731525883883 (+3 ms)Flushing b9ac03ed4dfd24d768bc705ca3d577cd/B: closing flushed file at 1731525883883Flushing b9ac03ed4dfd24d768bc705ca3d577cd/C: creating writer at 1731525884299 (+416 ms)Flushing b9ac03ed4dfd24d768bc705ca3d577cd/C: appending metadata at 1731525884302 (+3 ms)Flushing b9ac03ed4dfd24d768bc705ca3d577cd/C: closing flushed file at 1731525884302Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4446a37f: reopening flushed file at 1731525884718 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3517e01f: reopening flushed file at 1731525884723 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a7ed162: reopening flushed file at 1731525884727 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for b9ac03ed4dfd24d768bc705ca3d577cd in 1276ms, sequenceid=366, compaction requested=true at 1731525884731 (+4 ms)Writing region close event to WAL at 1731525884800 (+69 ms)Running coprocessor post-close hooks at 1731525884804 (+4 ms)Closed at 1731525884804 2024-11-13T19:24:44,806 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:44,806 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=b9ac03ed4dfd24d768bc705ca3d577cd, regionState=CLOSED 2024-11-13T19:24:44,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:24:44,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-13T19:24:44,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure b9ac03ed4dfd24d768bc705ca3d577cd, server=e4b56bc8c4b3,40371,1731525792389 in 1.5070 sec 2024-11-13T19:24:44,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-13T19:24:44,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b9ac03ed4dfd24d768bc705ca3d577cd, UNASSIGN in 1.5130 sec 2024-11-13T19:24:44,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-13T19:24:44,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5160 sec 2024-11-13T19:24:44,815 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525884815"}]},"ts":"1731525884815"} 2024-11-13T19:24:44,816 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:24:44,816 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:24:44,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5280 sec 2024-11-13T19:24:45,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-13T19:24:45,442 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:45,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:24:45,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:45,444 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:45,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T19:24:45,445 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:45,448 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:45,450 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/recovered.edits] 2024-11-13T19:24:45,454 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/38fb4925441d4e78bca12b38a7680a43 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/38fb4925441d4e78bca12b38a7680a43 2024-11-13T19:24:45,455 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/7330c7376be8480284d99250f6a99acc to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/7330c7376be8480284d99250f6a99acc 2024-11-13T19:24:45,457 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f48b2509ce12437b88d935f8819b3f88 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/A/f48b2509ce12437b88d935f8819b3f88 2024-11-13T19:24:45,459 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1996e11158cd461482f0a68737f2d916 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/1996e11158cd461482f0a68737f2d916 2024-11-13T19:24:45,460 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2d505584898496e889fee7ccffe3b3a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2d505584898496e889fee7ccffe3b3a 2024-11-13T19:24:45,461 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2ed3f894ea74b7bb4eea71c0f0d3b27 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/B/c2ed3f894ea74b7bb4eea71c0f0d3b27 2024-11-13T19:24:45,463 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/1064f2609273405e9697ef8e67480b20 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/1064f2609273405e9697ef8e67480b20 2024-11-13T19:24:45,464 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/68ec1d793d194622b8bdef3058d4c469 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/68ec1d793d194622b8bdef3058d4c469 2024-11-13T19:24:45,464 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/cae7544bee354814bfa7f6c4c279387e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/C/cae7544bee354814bfa7f6c4c279387e 2024-11-13T19:24:45,466 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/recovered.edits/369.seqid to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd/recovered.edits/369.seqid 2024-11-13T19:24:45,467 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b9ac03ed4dfd24d768bc705ca3d577cd 2024-11-13T19:24:45,467 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:24:45,468 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:45,472 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:24:45,474 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:24:45,475 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:45,475 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:24:45,475 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731525885475"}]},"ts":"9223372036854775807"} 2024-11-13T19:24:45,477 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:24:45,477 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b9ac03ed4dfd24d768bc705ca3d577cd, NAME => 'TestAcidGuarantees,,1731525853308.b9ac03ed4dfd24d768bc705ca3d577cd.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:24:45,477 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:24:45,477 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731525885477"}]},"ts":"9223372036854775807"} 2024-11-13T19:24:45,479 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:24:45,479 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:45,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 37 msec 2024-11-13T19:24:45,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-13T19:24:45,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:24:45,552 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:45,566 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 242), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1187 (was 1677), ProcessCount=11 (was 11), AvailableMemoryMB=2611 (was 238) - AvailableMemoryMB LEAK? - 2024-11-13T19:24:45,575 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=1187, ProcessCount=11, AvailableMemoryMB=2610 2024-11-13T19:24:45,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:24:45,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:24:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:45,578 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:24:45,578 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:45,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-13T19:24:45,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:24:45,579 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:24:45,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742138_1314 (size=959) 2024-11-13T19:24:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:24:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:24:45,991 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:24:45,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742139_1315 (size=53) 2024-11-13T19:24:46,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:24:46,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:24:46,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b04e7477f4d6e72d4397199d014ea80b, disabling compactions & flushes 2024-11-13T19:24:46,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:46,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:46,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. after waiting 0 ms 2024-11-13T19:24:46,402 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:46,402 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:46,403 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b04e7477f4d6e72d4397199d014ea80b: Waiting for close lock at 1731525886402Disabling compacts and flushes for region at 1731525886402Disabling writes for close at 1731525886402Writing region close event to WAL at 1731525886402Closed at 1731525886402 2024-11-13T19:24:46,405 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:24:46,406 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731525886405"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731525886405"}]},"ts":"1731525886405"} 2024-11-13T19:24:46,410 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:24:46,411 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:24:46,411 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525886411"}]},"ts":"1731525886411"} 2024-11-13T19:24:46,412 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:24:46,412 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, ASSIGN}] 2024-11-13T19:24:46,413 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, ASSIGN 2024-11-13T19:24:46,414 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, ASSIGN; state=OFFLINE, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=false 2024-11-13T19:24:46,565 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:46,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, ASSIGN because future has completed 2024-11-13T19:24:46,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:24:46,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:24:46,730 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:46,730 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:24:46,731 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,731 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:24:46,731 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,731 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,733 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,736 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b04e7477f4d6e72d4397199d014ea80b columnFamilyName A 2024-11-13T19:24:46,736 DEBUG [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:46,736 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(327): Store=b04e7477f4d6e72d4397199d014ea80b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:46,737 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,738 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b04e7477f4d6e72d4397199d014ea80b columnFamilyName B 2024-11-13T19:24:46,738 DEBUG [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:46,739 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(327): Store=b04e7477f4d6e72d4397199d014ea80b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:46,739 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,741 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b04e7477f4d6e72d4397199d014ea80b columnFamilyName C 2024-11-13T19:24:46,741 DEBUG [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:46,742 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(327): Store=b04e7477f4d6e72d4397199d014ea80b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:46,742 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,743 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,743 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,744 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,744 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,744 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:24:46,745 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,746 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:24:46,747 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened b04e7477f4d6e72d4397199d014ea80b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71779592, jitterRate=0.06959927082061768}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:24:46,747 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:46,747 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for b04e7477f4d6e72d4397199d014ea80b: Running coprocessor pre-open hook at 1731525886731Writing region info on filesystem at 1731525886731Initializing all the Stores at 1731525886733 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525886733Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525886733Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525886733Cleaning up temporary data from old regions at 1731525886744 (+11 ms)Running coprocessor post-open hooks at 1731525886747 (+3 ms)Region opened successfully at 1731525886747 2024-11-13T19:24:46,748 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., pid=89, masterSystemTime=1731525886724 2024-11-13T19:24:46,750 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:46,750 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:46,750 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=OPEN, openSeqNum=2, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:46,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:24:46,752 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44929 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=e4b56bc8c4b3,40371,1731525792389, table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T19:24:46,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-13T19:24:46,754 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 in 182 msec 2024-11-13T19:24:46,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-13T19:24:46,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, ASSIGN in 342 msec 2024-11-13T19:24:46,756 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:24:46,756 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525886756"}]},"ts":"1731525886756"} 2024-11-13T19:24:46,757 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:24:46,758 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:24:46,759 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-13T19:24:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-13T19:24:47,731 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:47,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197fefe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:47,733 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:47,733 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:47,734 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2774cc29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:47,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:47,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:47,737 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33272, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:47,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5280eeed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:47,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:47,741 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:47,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:47,743 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:24:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:24:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63d5be53 2024-11-13T19:24:47,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:24:47,745 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:24:47,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:24:47,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:24:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T19:24:47,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742140_1316 (size=995) 2024-11-13T19:24:48,162 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T19:24:48,162 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T19:24:48,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:24:48,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, REOPEN/MOVE}] 2024-11-13T19:24:48,169 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, REOPEN/MOVE 2024-11-13T19:24:48,169 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,170 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, REOPEN/MOVE because future has completed 2024-11-13T19:24:48,171 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:24:48,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:24:48,324 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,324 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:24:48,324 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing b04e7477f4d6e72d4397199d014ea80b, disabling compactions & flushes 2024-11-13T19:24:48,324 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,324 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,324 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. after waiting 0 ms 2024-11-13T19:24:48,324 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,333 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T19:24:48,334 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,334 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for b04e7477f4d6e72d4397199d014ea80b: Waiting for close lock at 1731525888324Running coprocessor pre-close hooks at 1731525888324Disabling compacts and flushes for region at 1731525888324Disabling writes for close at 1731525888324Writing region close event to WAL at 1731525888325 (+1 ms)Running coprocessor post-close hooks at 1731525888334 (+9 ms)Closed at 1731525888334 2024-11-13T19:24:48,334 WARN [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: b04e7477f4d6e72d4397199d014ea80b to self. 2024-11-13T19:24:48,337 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,338 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=CLOSED 2024-11-13T19:24:48,340 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:24:48,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-13T19:24:48,343 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 in 170 msec 2024-11-13T19:24:48,344 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, REOPEN/MOVE; state=CLOSED, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=true 2024-11-13T19:24:48,496 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, REOPEN/MOVE because future has completed 2024-11-13T19:24:48,499 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:24:48,659 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,660 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:24:48,660 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,660 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:24:48,660 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,660 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,662 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,663 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b04e7477f4d6e72d4397199d014ea80b columnFamilyName A 2024-11-13T19:24:48,664 DEBUG [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:48,665 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(327): Store=b04e7477f4d6e72d4397199d014ea80b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:48,665 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,666 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b04e7477f4d6e72d4397199d014ea80b columnFamilyName B 2024-11-13T19:24:48,666 DEBUG [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:48,666 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(327): Store=b04e7477f4d6e72d4397199d014ea80b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:48,666 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,667 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b04e7477f4d6e72d4397199d014ea80b columnFamilyName C 2024-11-13T19:24:48,667 DEBUG [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:48,668 INFO [StoreOpener-b04e7477f4d6e72d4397199d014ea80b-1 {}] regionserver.HStore(327): Store=b04e7477f4d6e72d4397199d014ea80b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:24:48,668 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,668 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,670 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,671 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,671 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,671 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:24:48,673 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,673 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened b04e7477f4d6e72d4397199d014ea80b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63756088, jitterRate=-0.04996025562286377}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:24:48,673 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,675 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for b04e7477f4d6e72d4397199d014ea80b: Running coprocessor pre-open hook at 1731525888660Writing region info on filesystem at 1731525888660Initializing all the Stores at 1731525888661 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525888662 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525888662Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525888662Cleaning up temporary data from old regions at 1731525888671 (+9 ms)Running coprocessor post-open hooks at 1731525888673 (+2 ms)Region opened successfully at 1731525888675 (+2 ms) 2024-11-13T19:24:48,676 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., pid=94, masterSystemTime=1731525888653 2024-11-13T19:24:48,678 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,678 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,679 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=OPEN, openSeqNum=5, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:24:48,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-13T19:24:48,684 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 in 183 msec 2024-11-13T19:24:48,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-13T19:24:48,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, REOPEN/MOVE in 516 msec 2024-11-13T19:24:48,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-13T19:24:48,688 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 521 msec 2024-11-13T19:24:48,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 942 msec 2024-11-13T19:24:48,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-13T19:24:48,691 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:48,691 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2f185f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,691 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,691 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,692 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75d10b12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,692 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb0d385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a739f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,695 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c38023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,697 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,698 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43909673, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,698 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@87fd48f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,698 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,699 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,699 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,699 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11bbc90f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@403d8b77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f0bd6a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,702 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,702 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,703 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3812251, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d609, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72c86b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,706 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,706 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@298e52a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,708 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77c30a8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23d098cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,709 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,710 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,710 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee985ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,712 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f0fa0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9deaeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,715 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a88b700, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,717 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54990, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,717 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3319d548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f8ca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,718 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,719 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31afceca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,721 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e12a064, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,722 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1515ed5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,722 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,723 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78f71261, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,723 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,724 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,724 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,725 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1edae097, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d10c554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:24:48,726 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:24:48,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:24:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:24:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:24:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218ab265, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:24:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:24:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:24:48,727 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:24:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9fc8fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:24:48,729 DEBUG [Thread-1453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,729 DEBUG [Thread-1464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,729 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,729 DEBUG [Thread-1459 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,729 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,729 DEBUG [Thread-1470 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,730 DEBUG [Thread-1451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,730 DEBUG [Thread-1455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,731 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,731 DEBUG [Thread-1457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:24:48,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,731 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:48,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,731 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,732 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57228, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:24:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:24:48,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-13T19:24:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:24:48,734 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:48,735 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,735 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:48,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:24:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,740 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:24:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:48,750 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:48,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:24:48,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b0946ff5420f4ff4b743772e4d451bd9_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525888750/Put/seqid=0 2024-11-13T19:24:48,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742141_1317 (size=14594) 2024-11-13T19:24:48,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525948811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525948819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525948821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525948822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525948823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:24:48,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:24:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:48,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:48,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:48,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525948931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525948942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525948942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525948942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525948942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:24:49,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:49,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:24:49,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525949142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525949151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525949151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525949151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525949151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,186 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b0946ff5420f4ff4b743772e4d451bd9_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b0946ff5420f4ff4b743772e4d451bd9_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:49,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4656978ee89c49528a30a1a4f6608ec0, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:49,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4656978ee89c49528a30a1a4f6608ec0 is 175, key is test_row_0/A:col10/1731525888750/Put/seqid=0 2024-11-13T19:24:49,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742142_1318 (size=39549) 2024-11-13T19:24:49,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:24:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:24:49,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:24:49,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525949452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525949462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525949462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525949462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:49,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525949462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:49,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:49,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:49,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:24:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4656978ee89c49528a30a1a4f6608ec0 2024-11-13T19:24:49,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/561403b86cea4a93927148ff5d9c0c72 is 50, key is test_row_0/B:col10/1731525888750/Put/seqid=0 2024-11-13T19:24:49,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742143_1319 (size=12001) 2024-11-13T19:24:49,617 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/561403b86cea4a93927148ff5d9c0c72 2024-11-13T19:24:49,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/3edbe1c57f6343b3b99bd6453bf8e51f is 50, key is test_row_0/C:col10/1731525888750/Put/seqid=0 2024-11-13T19:24:49,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:24:49,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:49,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:49,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742144_1320 (size=12001) 2024-11-13T19:24:49,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/3edbe1c57f6343b3b99bd6453bf8e51f 2024-11-13T19:24:49,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4656978ee89c49528a30a1a4f6608ec0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4656978ee89c49528a30a1a4f6608ec0 2024-11-13T19:24:49,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4656978ee89c49528a30a1a4f6608ec0, entries=200, sequenceid=16, filesize=38.6 K 2024-11-13T19:24:49,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/561403b86cea4a93927148ff5d9c0c72 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/561403b86cea4a93927148ff5d9c0c72 2024-11-13T19:24:49,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/561403b86cea4a93927148ff5d9c0c72, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:24:49,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/3edbe1c57f6343b3b99bd6453bf8e51f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3edbe1c57f6343b3b99bd6453bf8e51f 2024-11-13T19:24:49,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3edbe1c57f6343b3b99bd6453bf8e51f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-13T19:24:49,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b04e7477f4d6e72d4397199d014ea80b in 938ms, sequenceid=16, compaction requested=false 2024-11-13T19:24:49,688 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T19:24:49,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:49,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-13T19:24:49,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,809 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:24:49,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136aa8b07a242046ebaf5c461ca81b58f1_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525888820/Put/seqid=0 2024-11-13T19:24:49,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742145_1321 (size=12154) 2024-11-13T19:24:49,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,847 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411136aa8b07a242046ebaf5c461ca81b58f1_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136aa8b07a242046ebaf5c461ca81b58f1_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:49,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/44d68867118d41cf84f1554c246ba72d, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/44d68867118d41cf84f1554c246ba72d is 175, key is test_row_0/A:col10/1731525888820/Put/seqid=0 2024-11-13T19:24:49,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742146_1322 (size=30955) 2024-11-13T19:24:49,867 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/44d68867118d41cf84f1554c246ba72d 2024-11-13T19:24:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:24:49,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ab6ed6325d324f84ae2534fb8fc883a0 is 50, key is test_row_0/B:col10/1731525888820/Put/seqid=0 2024-11-13T19:24:49,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742147_1323 (size=12001) 2024-11-13T19:24:49,888 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ab6ed6325d324f84ae2534fb8fc883a0 2024-11-13T19:24:49,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/206328ab78834bd2842052d44c3369ca is 50, key is test_row_0/C:col10/1731525888820/Put/seqid=0 2024-11-13T19:24:49,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742148_1324 (size=12001) 2024-11-13T19:24:49,908 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/206328ab78834bd2842052d44c3369ca 2024-11-13T19:24:49,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/44d68867118d41cf84f1554c246ba72d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/44d68867118d41cf84f1554c246ba72d 2024-11-13T19:24:49,917 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/44d68867118d41cf84f1554c246ba72d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T19:24:49,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ab6ed6325d324f84ae2534fb8fc883a0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ab6ed6325d324f84ae2534fb8fc883a0 2024-11-13T19:24:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,921 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ab6ed6325d324f84ae2534fb8fc883a0, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T19:24:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/206328ab78834bd2842052d44c3369ca as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/206328ab78834bd2842052d44c3369ca 2024-11-13T19:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,924 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/206328ab78834bd2842052d44c3369ca, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T19:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,927 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for b04e7477f4d6e72d4397199d014ea80b in 117ms, sequenceid=40, compaction requested=false 2024-11-13T19:24:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:49,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-13T19:24:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-13T19:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-13T19:24:49,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1930 sec 2024-11-13T19:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,933 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.2000 sec 2024-11-13T19:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:50,051 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:50,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:24:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113af48940845cb497897679f66c4ae6366_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525890038/Put/seqid=0 2024-11-13T19:24:50,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742150_1326 (size=26798) 2024-11-13T19:24:50,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525950128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525950134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525950138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525950139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525950140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,150 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525950241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,246 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525950251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525950252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525950252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525950252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,334 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:24:50,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525950451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525950461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525950471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525950472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525950472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,485 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,492 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:50,495 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113af48940845cb497897679f66c4ae6366_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af48940845cb497897679f66c4ae6366_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:50,496 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4853e46af4b3400e924ad1cc8b4f3975, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:50,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4853e46af4b3400e924ad1cc8b4f3975 is 175, key is test_row_0/A:col10/1731525890038/Put/seqid=0 2024-11-13T19:24:50,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742149_1325 (size=82585) 2024-11-13T19:24:50,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4853e46af4b3400e924ad1cc8b4f3975 2024-11-13T19:24:50,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/2bc3f47b56d44dc38d1cd9eb6e69282a is 50, key is test_row_0/B:col10/1731525890038/Put/seqid=0 2024-11-13T19:24:50,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742151_1327 (size=12001) 2024-11-13T19:24:50,520 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/2bc3f47b56d44dc38d1cd9eb6e69282a 2024-11-13T19:24:50,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/f3d6fea95c6b476a9c6ddf39a814cba6 is 50, key is test_row_0/C:col10/1731525890038/Put/seqid=0 2024-11-13T19:24:50,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742152_1328 (size=12001) 2024-11-13T19:24:50,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525950761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525950781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525950791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525950791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525950792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:50,800 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:50,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-13T19:24:50,882 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:50,882 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-13T19:24:50,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:24:50,884 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:50,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:50,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/f3d6fea95c6b476a9c6ddf39a814cba6 2024-11-13T19:24:50,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/4853e46af4b3400e924ad1cc8b4f3975 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4853e46af4b3400e924ad1cc8b4f3975 2024-11-13T19:24:50,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4853e46af4b3400e924ad1cc8b4f3975, entries=450, sequenceid=52, filesize=80.6 K 2024-11-13T19:24:50,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/2bc3f47b56d44dc38d1cd9eb6e69282a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/2bc3f47b56d44dc38d1cd9eb6e69282a 2024-11-13T19:24:50,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/2bc3f47b56d44dc38d1cd9eb6e69282a, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T19:24:50,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/f3d6fea95c6b476a9c6ddf39a814cba6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f3d6fea95c6b476a9c6ddf39a814cba6 2024-11-13T19:24:50,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f3d6fea95c6b476a9c6ddf39a814cba6, entries=150, sequenceid=52, filesize=11.7 K 2024-11-13T19:24:50,952 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-13T19:24:50,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b04e7477f4d6e72d4397199d014ea80b in 902ms, sequenceid=52, compaction requested=true 2024-11-13T19:24:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:50,953 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:50,953 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:50,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:50,954 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:50,954 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 153089 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:50,954 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/B is initiating minor compaction (all files) 2024-11-13T19:24:50,954 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/A is initiating minor compaction (all files) 2024-11-13T19:24:50,954 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/B in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:50,954 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/A in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:50,954 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/561403b86cea4a93927148ff5d9c0c72, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ab6ed6325d324f84ae2534fb8fc883a0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/2bc3f47b56d44dc38d1cd9eb6e69282a] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=35.2 K 2024-11-13T19:24:50,954 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4656978ee89c49528a30a1a4f6608ec0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/44d68867118d41cf84f1554c246ba72d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4853e46af4b3400e924ad1cc8b4f3975] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=149.5 K 2024-11-13T19:24:50,954 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:50,954 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4656978ee89c49528a30a1a4f6608ec0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/44d68867118d41cf84f1554c246ba72d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4853e46af4b3400e924ad1cc8b4f3975] 2024-11-13T19:24:50,955 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4656978ee89c49528a30a1a4f6608ec0, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731525888738 2024-11-13T19:24:50,955 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 561403b86cea4a93927148ff5d9c0c72, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731525888738 2024-11-13T19:24:50,955 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6ed6325d324f84ae2534fb8fc883a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525888786 2024-11-13T19:24:50,955 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44d68867118d41cf84f1554c246ba72d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525888786 2024-11-13T19:24:50,955 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bc3f47b56d44dc38d1cd9eb6e69282a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525890038 2024-11-13T19:24:50,955 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4853e46af4b3400e924ad1cc8b4f3975, keycount=450, bloomtype=ROW, size=80.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525889998 2024-11-13T19:24:50,961 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#B#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:50,961 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/8c40625900a9412aa3402753de8d2243 is 50, key is test_row_0/B:col10/1731525890038/Put/seqid=0 2024-11-13T19:24:50,962 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:50,967 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111379c8a00e94c44a1b873c6da2ca73104f_b04e7477f4d6e72d4397199d014ea80b store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:50,970 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111379c8a00e94c44a1b873c6da2ca73104f_b04e7477f4d6e72d4397199d014ea80b, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:50,970 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379c8a00e94c44a1b873c6da2ca73104f_b04e7477f4d6e72d4397199d014ea80b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:50,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742153_1329 (size=12104) 2024-11-13T19:24:50,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742154_1330 (size=4469) 2024-11-13T19:24:50,983 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#A#compaction#283 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:50,984 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/67fe23053d244263923655e747b4e2a9 is 175, key is test_row_0/A:col10/1731525890038/Put/seqid=0 2024-11-13T19:24:50,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:24:50,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742155_1331 (size=31058) 2024-11-13T19:24:51,005 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/67fe23053d244263923655e747b4e2a9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/67fe23053d244263923655e747b4e2a9 2024-11-13T19:24:51,009 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/A of b04e7477f4d6e72d4397199d014ea80b into 67fe23053d244263923655e747b4e2a9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:51,009 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:51,009 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/A, priority=13, startTime=1731525890953; duration=0sec 2024-11-13T19:24:51,009 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:51,009 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:A 2024-11-13T19:24:51,009 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:51,010 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:51,010 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/C is initiating minor compaction (all files) 2024-11-13T19:24:51,010 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/C in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:51,010 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3edbe1c57f6343b3b99bd6453bf8e51f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/206328ab78834bd2842052d44c3369ca, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f3d6fea95c6b476a9c6ddf39a814cba6] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=35.2 K 2024-11-13T19:24:51,010 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3edbe1c57f6343b3b99bd6453bf8e51f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731525888738 2024-11-13T19:24:51,011 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 206328ab78834bd2842052d44c3369ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525888786 2024-11-13T19:24:51,011 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d6fea95c6b476a9c6ddf39a814cba6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525890038 2024-11-13T19:24:51,017 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#C#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:51,017 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/e2b75925c19f4e91ac8e7ac3c4dc066a is 50, key is test_row_0/C:col10/1731525890038/Put/seqid=0 2024-11-13T19:24:51,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742156_1332 (size=12104) 2024-11-13T19:24:51,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-13T19:24:51,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:51,037 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:24:51,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c75e263d275c4816b82c10a99e66bb6b_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525890127/Put/seqid=0 2024-11-13T19:24:51,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742157_1333 (size=12154) 2024-11-13T19:24:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:24:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:51,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:51,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525951306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525951311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525951311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525951311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525951313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,375 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/8c40625900a9412aa3402753de8d2243 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/8c40625900a9412aa3402753de8d2243 2024-11-13T19:24:51,379 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/B of b04e7477f4d6e72d4397199d014ea80b into 8c40625900a9412aa3402753de8d2243(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:51,379 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:51,379 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/B, priority=13, startTime=1731525890953; duration=0sec 2024-11-13T19:24:51,379 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:51,379 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:B 2024-11-13T19:24:51,425 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/e2b75925c19f4e91ac8e7ac3c4dc066a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/e2b75925c19f4e91ac8e7ac3c4dc066a 2024-11-13T19:24:51,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525951421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525951421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,429 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/C of b04e7477f4d6e72d4397199d014ea80b into e2b75925c19f4e91ac8e7ac3c4dc066a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:51,429 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:51,429 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/C, priority=13, startTime=1731525890953; duration=0sec 2024-11-13T19:24:51,430 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:51,430 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:C 2024-11-13T19:24:51,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:51,452 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c75e263d275c4816b82c10a99e66bb6b_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c75e263d275c4816b82c10a99e66bb6b_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:51,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/3ac935fcdf5847958d67466ff1a3ff4b, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:51,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/3ac935fcdf5847958d67466ff1a3ff4b is 175, key is test_row_0/A:col10/1731525890127/Put/seqid=0 2024-11-13T19:24:51,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742158_1334 (size=30955) 2024-11-13T19:24:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:24:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525951631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525951631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,857 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/3ac935fcdf5847958d67466ff1a3ff4b 2024-11-13T19:24:51,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/894236f841b14534b8a3715f3fa58111 is 50, key is test_row_0/B:col10/1731525890127/Put/seqid=0 2024-11-13T19:24:51,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742159_1335 (size=12001) 2024-11-13T19:24:51,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525951942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525951942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:51,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:52,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:24:52,270 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/894236f841b14534b8a3715f3fa58111 2024-11-13T19:24:52,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/6ee5ae9e4abd4f589caac1c83fc1e371 is 50, key is test_row_0/C:col10/1731525890127/Put/seqid=0 2024-11-13T19:24:52,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742160_1336 (size=12001) 2024-11-13T19:24:52,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525952322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:52,328 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:52,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525952331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:52,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525952331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:52,336 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:52,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:52,336 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:52,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525952451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:52,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525952451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:52,454 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:52,680 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/6ee5ae9e4abd4f589caac1c83fc1e371 2024-11-13T19:24:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/3ac935fcdf5847958d67466ff1a3ff4b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/3ac935fcdf5847958d67466ff1a3ff4b 2024-11-13T19:24:52,687 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/3ac935fcdf5847958d67466ff1a3ff4b, entries=150, sequenceid=77, filesize=30.2 K 2024-11-13T19:24:52,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/894236f841b14534b8a3715f3fa58111 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/894236f841b14534b8a3715f3fa58111 2024-11-13T19:24:52,691 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/894236f841b14534b8a3715f3fa58111, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T19:24:52,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/6ee5ae9e4abd4f589caac1c83fc1e371 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6ee5ae9e4abd4f589caac1c83fc1e371 2024-11-13T19:24:52,696 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6ee5ae9e4abd4f589caac1c83fc1e371, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T19:24:52,697 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b04e7477f4d6e72d4397199d014ea80b in 1661ms, sequenceid=77, compaction requested=false 2024-11-13T19:24:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:52,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-13T19:24:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-13T19:24:52,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-13T19:24:52,700 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8140 sec 2024-11-13T19:24:52,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8190 sec 2024-11-13T19:24:53,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-13T19:24:53,042 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:53,042 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:53,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-13T19:24:53,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:24:53,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:53,044 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:53,045 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:24:53,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-13T19:24:53,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:53,197 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-13T19:24:53,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111317a23186c6114935b0f4d773d04f52dd_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525891302/Put/seqid=0 2024-11-13T19:24:53,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742161_1337 (size=12154) 2024-11-13T19:24:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:24:53,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:53,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:53,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:53,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525953538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:53,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:53,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525953547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:53,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:53,611 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111317a23186c6114935b0f4d773d04f52dd_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111317a23186c6114935b0f4d773d04f52dd_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/a153f638afa8470e9a65bb32c90b1766, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:53,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/a153f638afa8470e9a65bb32c90b1766 is 175, key is test_row_0/A:col10/1731525891302/Put/seqid=0 2024-11-13T19:24:53,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742162_1338 (size=30955) 2024-11-13T19:24:53,617 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/a153f638afa8470e9a65bb32c90b1766 2024-11-13T19:24:53,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/a27c883267b945daa690a0b4ef163772 is 50, key is test_row_0/B:col10/1731525891302/Put/seqid=0 2024-11-13T19:24:53,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742163_1339 (size=12001) 2024-11-13T19:24:53,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:53,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525953651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:53,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:53,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525953661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:53,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:24:53,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525953861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:53,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:53,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525953871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:53,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,027 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/a27c883267b945daa690a0b4ef163772 2024-11-13T19:24:54,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/ca0d8f63759646d0a9773be8db805750 is 50, key is test_row_0/C:col10/1731525891302/Put/seqid=0 2024-11-13T19:24:54,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742164_1340 (size=12001) 2024-11-13T19:24:54,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525954172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:24:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525954181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,187 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525954341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,347 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525954352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,356 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525954361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,368 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,368 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,438 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/ca0d8f63759646d0a9773be8db805750 2024-11-13T19:24:54,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/a153f638afa8470e9a65bb32c90b1766 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/a153f638afa8470e9a65bb32c90b1766 2024-11-13T19:24:54,446 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/a153f638afa8470e9a65bb32c90b1766, entries=150, sequenceid=91, filesize=30.2 K 2024-11-13T19:24:54,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/a27c883267b945daa690a0b4ef163772 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a27c883267b945daa690a0b4ef163772 2024-11-13T19:24:54,450 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a27c883267b945daa690a0b4ef163772, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T19:24:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/ca0d8f63759646d0a9773be8db805750 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/ca0d8f63759646d0a9773be8db805750 2024-11-13T19:24:54,454 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/ca0d8f63759646d0a9773be8db805750, entries=150, sequenceid=91, filesize=11.7 K 2024-11-13T19:24:54,455 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b04e7477f4d6e72d4397199d014ea80b in 1258ms, sequenceid=91, compaction requested=true 2024-11-13T19:24:54,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:54,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:54,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-13T19:24:54,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-13T19:24:54,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-13T19:24:54,458 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4130 sec 2024-11-13T19:24:54,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4170 sec 2024-11-13T19:24:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:54,686 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:54,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T19:24:54,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e94fce7fc68f4fdf8ad615c085c44351_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525893546/Put/seqid=0 2024-11-13T19:24:54,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742165_1341 (size=14594) 2024-11-13T19:24:54,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525954715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525954716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525954832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,839 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525954832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:54,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525955041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,046 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:55,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:55,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525955051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:55,096 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:55,099 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e94fce7fc68f4fdf8ad615c085c44351_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e94fce7fc68f4fdf8ad615c085c44351_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:55,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0717adac96ce4b229c25ad339cf5426c, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:55,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0717adac96ce4b229c25ad339cf5426c is 175, key is test_row_0/A:col10/1731525893546/Put/seqid=0 2024-11-13T19:24:55,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742166_1342 (size=39549) 2024-11-13T19:24:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-13T19:24:55,192 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:55,192 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-13T19:24:55,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:24:55,194 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:55,195 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:55,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:55,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:24:55,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525955351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:55,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525955361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:55,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,504 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0717adac96ce4b229c25ad339cf5426c 2024-11-13T19:24:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:24:55,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/6e988d2562c64bc18cec9880f17e5537 is 50, key is test_row_0/B:col10/1731525893546/Put/seqid=0 2024-11-13T19:24:55,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742167_1343 (size=12001) 2024-11-13T19:24:55,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:55,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:55,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:24:55,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525955871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:55,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525955871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:55,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:55,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/6e988d2562c64bc18cec9880f17e5537 2024-11-13T19:24:55,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/858d172942ce4666b5e766733e8571e1 is 50, key is test_row_0/C:col10/1731525893546/Put/seqid=0 2024-11-13T19:24:55,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742168_1344 (size=12001) 2024-11-13T19:24:55,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:55,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:55,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:56,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:56,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:56,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:56,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,263 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:56,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:56,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/858d172942ce4666b5e766733e8571e1 2024-11-13T19:24:56,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:24:56,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0717adac96ce4b229c25ad339cf5426c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0717adac96ce4b229c25ad339cf5426c 2024-11-13T19:24:56,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0717adac96ce4b229c25ad339cf5426c, entries=200, sequenceid=116, filesize=38.6 K 2024-11-13T19:24:56,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/6e988d2562c64bc18cec9880f17e5537 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/6e988d2562c64bc18cec9880f17e5537 2024-11-13T19:24:56,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/6e988d2562c64bc18cec9880f17e5537, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T19:24:56,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/858d172942ce4666b5e766733e8571e1 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/858d172942ce4666b5e766733e8571e1 2024-11-13T19:24:56,344 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/858d172942ce4666b5e766733e8571e1, entries=150, sequenceid=116, filesize=11.7 K 2024-11-13T19:24:56,345 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b04e7477f4d6e72d4397199d014ea80b in 1659ms, sequenceid=116, compaction requested=true 2024-11-13T19:24:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:56,345 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:56,345 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:56,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:56,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:56,346 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:56,346 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132517 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:56,346 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/A is initiating minor compaction (all files) 2024-11-13T19:24:56,346 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/B is initiating minor compaction (all files) 2024-11-13T19:24:56,346 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/B in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,346 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/A in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,346 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/8c40625900a9412aa3402753de8d2243, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/894236f841b14534b8a3715f3fa58111, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a27c883267b945daa690a0b4ef163772, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/6e988d2562c64bc18cec9880f17e5537] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=47.0 K 2024-11-13T19:24:56,346 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/67fe23053d244263923655e747b4e2a9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/3ac935fcdf5847958d67466ff1a3ff4b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/a153f638afa8470e9a65bb32c90b1766, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0717adac96ce4b229c25ad339cf5426c] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=129.4 K 2024-11-13T19:24:56,347 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,347 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/67fe23053d244263923655e747b4e2a9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/3ac935fcdf5847958d67466ff1a3ff4b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/a153f638afa8470e9a65bb32c90b1766, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0717adac96ce4b229c25ad339cf5426c] 2024-11-13T19:24:56,347 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c40625900a9412aa3402753de8d2243, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525890038 2024-11-13T19:24:56,347 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67fe23053d244263923655e747b4e2a9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525890038 2024-11-13T19:24:56,347 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ac935fcdf5847958d67466ff1a3ff4b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525890114 2024-11-13T19:24:56,347 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 894236f841b14534b8a3715f3fa58111, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525890114 2024-11-13T19:24:56,347 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a153f638afa8470e9a65bb32c90b1766, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731525891302 2024-11-13T19:24:56,347 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a27c883267b945daa690a0b4ef163772, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731525891302 2024-11-13T19:24:56,348 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e988d2562c64bc18cec9880f17e5537, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731525893528 2024-11-13T19:24:56,348 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0717adac96ce4b229c25ad339cf5426c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731525893528 2024-11-13T19:24:56,352 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:56,353 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:56,354 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113ef247b1d3e744dd0b72f8d86dacfb6e3_b04e7477f4d6e72d4397199d014ea80b store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:56,354 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/584babef65fc42dc9a6b83625bf2ff89 is 50, key is test_row_0/B:col10/1731525893546/Put/seqid=0 2024-11-13T19:24:56,356 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113ef247b1d3e744dd0b72f8d86dacfb6e3_b04e7477f4d6e72d4397199d014ea80b, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:56,356 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113ef247b1d3e744dd0b72f8d86dacfb6e3_b04e7477f4d6e72d4397199d014ea80b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:56,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742169_1345 (size=12241) 2024-11-13T19:24:56,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742170_1346 (size=4469) 2024-11-13T19:24:56,370 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/584babef65fc42dc9a6b83625bf2ff89 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/584babef65fc42dc9a6b83625bf2ff89 2024-11-13T19:24:56,374 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/B of b04e7477f4d6e72d4397199d014ea80b into 584babef65fc42dc9a6b83625bf2ff89(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:56,374 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:56,374 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/B, priority=12, startTime=1731525896345; duration=0sec 2024-11-13T19:24:56,374 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:56,374 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:B 2024-11-13T19:24:56,374 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:24:56,375 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:24:56,375 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/C is initiating minor compaction (all files) 2024-11-13T19:24:56,376 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/C in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,376 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/e2b75925c19f4e91ac8e7ac3c4dc066a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6ee5ae9e4abd4f589caac1c83fc1e371, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/ca0d8f63759646d0a9773be8db805750, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/858d172942ce4666b5e766733e8571e1] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=47.0 K 2024-11-13T19:24:56,376 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e2b75925c19f4e91ac8e7ac3c4dc066a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731525890038 2024-11-13T19:24:56,376 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ee5ae9e4abd4f589caac1c83fc1e371, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525890114 2024-11-13T19:24:56,376 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting ca0d8f63759646d0a9773be8db805750, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1731525891302 2024-11-13T19:24:56,377 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 858d172942ce4666b5e766733e8571e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731525893528 2024-11-13T19:24:56,384 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:56,384 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/fe65953fc57148f09e65637c792d9a2f is 50, key is test_row_0/C:col10/1731525893546/Put/seqid=0 2024-11-13T19:24:56,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742171_1347 (size=12241) 2024-11-13T19:24:56,393 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/fe65953fc57148f09e65637c792d9a2f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/fe65953fc57148f09e65637c792d9a2f 2024-11-13T19:24:56,396 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/C of b04e7477f4d6e72d4397199d014ea80b into fe65953fc57148f09e65637c792d9a2f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:56,396 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:56,397 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/C, priority=12, startTime=1731525896346; duration=0sec 2024-11-13T19:24:56,397 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:56,397 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:C 2024-11-13T19:24:56,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-13T19:24:56,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:56,416 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:24:56,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111320e2ba76c8f5450ea5d6ffd7d297ffad_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525894700/Put/seqid=0 2024-11-13T19:24:56,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742172_1348 (size=12154) 2024-11-13T19:24:56,761 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#A#compaction#295 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:56,762 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/820cc6b532fd461494654016310738cd is 175, key is test_row_0/A:col10/1731525893546/Put/seqid=0 2024-11-13T19:24:56,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742173_1349 (size=31195) 2024-11-13T19:24:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:56,832 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111320e2ba76c8f5450ea5d6ffd7d297ffad_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111320e2ba76c8f5450ea5d6ffd7d297ffad_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/99c983306bfe4ce5ae53caad299a3f06, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:56,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/99c983306bfe4ce5ae53caad299a3f06 is 175, key is test_row_0/A:col10/1731525894700/Put/seqid=0 2024-11-13T19:24:56,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742174_1350 (size=30955) 2024-11-13T19:24:56,846 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/99c983306bfe4ce5ae53caad299a3f06 2024-11-13T19:24:56,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/d29916a7f06c4f0d861829e8ce017da3 is 50, key is test_row_0/B:col10/1731525894700/Put/seqid=0 2024-11-13T19:24:56,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742175_1351 (size=12001) 2024-11-13T19:24:56,880 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/d29916a7f06c4f0d861829e8ce017da3 2024-11-13T19:24:56,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/4e7b3b2904e04eadaf026faceb6df564 is 50, key is test_row_0/C:col10/1731525894700/Put/seqid=0 2024-11-13T19:24:56,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742176_1352 (size=12001) 2024-11-13T19:24:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:56,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:56,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525956981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:56,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:56,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:56,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:56,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525956986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:56,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525957101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,105 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525957101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,171 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/820cc6b532fd461494654016310738cd as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/820cc6b532fd461494654016310738cd 2024-11-13T19:24:57,175 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/A of b04e7477f4d6e72d4397199d014ea80b into 820cc6b532fd461494654016310738cd(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:57,175 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:57,175 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/A, priority=12, startTime=1731525896345; duration=0sec 2024-11-13T19:24:57,175 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:57,175 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:A 2024-11-13T19:24:57,295 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/4e7b3b2904e04eadaf026faceb6df564 2024-11-13T19:24:57,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/99c983306bfe4ce5ae53caad299a3f06 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/99c983306bfe4ce5ae53caad299a3f06 2024-11-13T19:24:57,303 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/99c983306bfe4ce5ae53caad299a3f06, entries=150, sequenceid=129, filesize=30.2 K 2024-11-13T19:24:57,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/d29916a7f06c4f0d861829e8ce017da3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/d29916a7f06c4f0d861829e8ce017da3 2024-11-13T19:24:57,306 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/d29916a7f06c4f0d861829e8ce017da3, entries=150, sequenceid=129, filesize=11.7 K 2024-11-13T19:24:57,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/4e7b3b2904e04eadaf026faceb6df564 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4e7b3b2904e04eadaf026faceb6df564 2024-11-13T19:24:57,310 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4e7b3b2904e04eadaf026faceb6df564, entries=150, sequenceid=129, filesize=11.7 K 2024-11-13T19:24:57,311 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for b04e7477f4d6e72d4397199d014ea80b in 896ms, sequenceid=129, compaction requested=false 2024-11-13T19:24:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-13T19:24:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-13T19:24:57,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-13T19:24:57,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1180 sec 2024-11-13T19:24:57,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1230 sec 2024-11-13T19:24:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:57,317 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:24:57,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-13T19:24:57,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a9fe6d782a174eb6ac949442cfab7d47_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525896979/Put/seqid=0 2024-11-13T19:24:57,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742177_1353 (size=14794) 2024-11-13T19:24:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-13T19:24:57,342 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:24:57,342 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:24:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-13T19:24:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T19:24:57,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:24:57,344 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:24:57,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:24:57,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525957339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525957339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T19:24:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525957452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525957452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:57,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:57,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T19:24:57,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525957661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525957661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,730 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:57,734 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a9fe6d782a174eb6ac949442cfab7d47_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a9fe6d782a174eb6ac949442cfab7d47_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:57,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/27eba7ccc644493bbe400072c8e5b852, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:57,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/27eba7ccc644493bbe400072c8e5b852 is 175, key is test_row_0/A:col10/1731525896979/Put/seqid=0 2024-11-13T19:24:57,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742178_1354 (size=39749) 2024-11-13T19:24:57,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:57,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:57,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:57,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:57,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:57,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T19:24:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525957972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525957972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:58,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:58,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,145 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/27eba7ccc644493bbe400072c8e5b852 2024-11-13T19:24:58,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/cce8e028063f4025915b7d9e87bd0825 is 50, key is test_row_0/B:col10/1731525896979/Put/seqid=0 2024-11-13T19:24:58,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742179_1355 (size=12151) 2024-11-13T19:24:58,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525958361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:58,367 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,368 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:58,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525958391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:58,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525958391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:58,398 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,398 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:24:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:58,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T19:24:58,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525958491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525958492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:58,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/cce8e028063f4025915b7d9e87bd0825 2024-11-13T19:24:58,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/77d1e94eadcd43fbac33f4af10957839 is 50, key is test_row_0/C:col10/1731525896979/Put/seqid=0 2024-11-13T19:24:58,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:58,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:58,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742180_1356 (size=12151) 2024-11-13T19:24:58,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:24:58,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/77d1e94eadcd43fbac33f4af10957839 2024-11-13T19:24:58,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/27eba7ccc644493bbe400072c8e5b852 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/27eba7ccc644493bbe400072c8e5b852 2024-11-13T19:24:58,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/27eba7ccc644493bbe400072c8e5b852, entries=200, sequenceid=156, filesize=38.8 K 2024-11-13T19:24:58,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/cce8e028063f4025915b7d9e87bd0825 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/cce8e028063f4025915b7d9e87bd0825 2024-11-13T19:24:58,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/cce8e028063f4025915b7d9e87bd0825, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T19:24:58,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/77d1e94eadcd43fbac33f4af10957839 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77d1e94eadcd43fbac33f4af10957839 2024-11-13T19:24:58,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77d1e94eadcd43fbac33f4af10957839, entries=150, sequenceid=156, filesize=11.9 K 2024-11-13T19:24:58,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for b04e7477f4d6e72d4397199d014ea80b in 1671ms, sequenceid=156, compaction requested=true 2024-11-13T19:24:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:24:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:24:58,988 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:58,988 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:24:58,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:58,989 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:58,989 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:58,989 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/A is initiating minor compaction (all files) 2024-11-13T19:24:58,989 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/B is initiating minor compaction (all files) 2024-11-13T19:24:58,989 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/A in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,989 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/B in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,989 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/584babef65fc42dc9a6b83625bf2ff89, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/d29916a7f06c4f0d861829e8ce017da3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/cce8e028063f4025915b7d9e87bd0825] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=35.5 K 2024-11-13T19:24:58,989 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/820cc6b532fd461494654016310738cd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/99c983306bfe4ce5ae53caad299a3f06, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/27eba7ccc644493bbe400072c8e5b852] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=99.5 K 2024-11-13T19:24:58,989 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:58,989 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/820cc6b532fd461494654016310738cd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/99c983306bfe4ce5ae53caad299a3f06, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/27eba7ccc644493bbe400072c8e5b852] 2024-11-13T19:24:58,990 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 584babef65fc42dc9a6b83625bf2ff89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731525893528 2024-11-13T19:24:58,990 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 820cc6b532fd461494654016310738cd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731525893528 2024-11-13T19:24:58,990 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting d29916a7f06c4f0d861829e8ce017da3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525894700 2024-11-13T19:24:58,990 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99c983306bfe4ce5ae53caad299a3f06, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525894700 2024-11-13T19:24:58,990 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting cce8e028063f4025915b7d9e87bd0825, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525896917 2024-11-13T19:24:58,990 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27eba7ccc644493bbe400072c8e5b852, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525896917 2024-11-13T19:24:58,995 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:58,996 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#B#compaction#303 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:58,996 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ff013bc5d0a345d2a16f7547defcee6b is 50, key is test_row_0/B:col10/1731525896979/Put/seqid=0 2024-11-13T19:24:58,997 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411139d762514241c43019cc5279308d2e551_b04e7477f4d6e72d4397199d014ea80b store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:58,999 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411139d762514241c43019cc5279308d2e551_b04e7477f4d6e72d4397199d014ea80b, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:58,999 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411139d762514241c43019cc5279308d2e551_b04e7477f4d6e72d4397199d014ea80b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:59,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742181_1357 (size=12493) 2024-11-13T19:24:59,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742182_1358 (size=4469) 2024-11-13T19:24:59,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-13T19:24:59,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:59,024 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-13T19:24:59,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113927fd666564947ef8788cabceaa607cc_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525897337/Put/seqid=0 2024-11-13T19:24:59,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742183_1359 (size=12304) 2024-11-13T19:24:59,405 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#A#compaction#304 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:59,405 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0367befe73b34c4599c47328b0774e34 is 175, key is test_row_0/A:col10/1731525896979/Put/seqid=0 2024-11-13T19:24:59,407 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ff013bc5d0a345d2a16f7547defcee6b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ff013bc5d0a345d2a16f7547defcee6b 2024-11-13T19:24:59,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742184_1360 (size=31447) 2024-11-13T19:24:59,411 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/B of b04e7477f4d6e72d4397199d014ea80b into ff013bc5d0a345d2a16f7547defcee6b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:59,412 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:59,412 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/B, priority=13, startTime=1731525898988; duration=0sec 2024-11-13T19:24:59,412 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:24:59,412 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:B 2024-11-13T19:24:59,412 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:24:59,413 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:24:59,413 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/C is initiating minor compaction (all files) 2024-11-13T19:24:59,413 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/C in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:24:59,413 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/fe65953fc57148f09e65637c792d9a2f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4e7b3b2904e04eadaf026faceb6df564, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77d1e94eadcd43fbac33f4af10957839] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=35.5 K 2024-11-13T19:24:59,413 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting fe65953fc57148f09e65637c792d9a2f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731525893528 2024-11-13T19:24:59,414 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7b3b2904e04eadaf026faceb6df564, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525894700 2024-11-13T19:24:59,414 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 77d1e94eadcd43fbac33f4af10957839, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525896917 2024-11-13T19:24:59,420 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#C#compaction#306 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:24:59,420 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/f766ad121f7e4e68bae8193030efa1b7 is 50, key is test_row_0/C:col10/1731525896979/Put/seqid=0 2024-11-13T19:24:59,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742185_1361 (size=12493) 2024-11-13T19:24:59,428 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/f766ad121f7e4e68bae8193030efa1b7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f766ad121f7e4e68bae8193030efa1b7 2024-11-13T19:24:59,433 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/C of b04e7477f4d6e72d4397199d014ea80b into f766ad121f7e4e68bae8193030efa1b7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:59,433 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:59,433 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/C, priority=13, startTime=1731525898988; duration=0sec 2024-11-13T19:24:59,433 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:59,433 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:C 2024-11-13T19:24:59,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:24:59,439 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113927fd666564947ef8788cabceaa607cc_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113927fd666564947ef8788cabceaa607cc_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:59,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/f294b92fc5ba478097821561f4a3b5b5, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:24:59,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/f294b92fc5ba478097821561f4a3b5b5 is 175, key is test_row_0/A:col10/1731525897337/Put/seqid=0 2024-11-13T19:24:59,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742186_1362 (size=31105) 2024-11-13T19:24:59,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T19:24:59,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:24:59,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:24:59,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:59,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525959591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:59,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:59,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525959596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:59,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:59,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525959701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:59,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:59,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525959711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:59,815 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0367befe73b34c4599c47328b0774e34 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0367befe73b34c4599c47328b0774e34 2024-11-13T19:24:59,819 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/A of b04e7477f4d6e72d4397199d014ea80b into 0367befe73b34c4599c47328b0774e34(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:24:59,819 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:24:59,819 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/A, priority=13, startTime=1731525898988; duration=0sec 2024-11-13T19:24:59,819 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:24:59,819 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:A 2024-11-13T19:24:59,846 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/f294b92fc5ba478097821561f4a3b5b5 2024-11-13T19:24:59,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/67710ec385a9404e8729eb3b5a0d5dd5 is 50, key is test_row_0/B:col10/1731525897337/Put/seqid=0 2024-11-13T19:24:59,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742187_1363 (size=12151) 2024-11-13T19:24:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525959912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:24:59,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:24:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525959921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:24:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:24:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:00,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:00,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525960222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,228 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:00,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:00,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525960231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:00,255 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/67710ec385a9404e8729eb3b5a0d5dd5 2024-11-13T19:25:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0df4a2ae32d04bf8b358b62e02f1b7e6 is 50, key is test_row_0/C:col10/1731525897337/Put/seqid=0 2024-11-13T19:25:00,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742188_1364 (size=12151) 2024-11-13T19:25:00,664 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0df4a2ae32d04bf8b358b62e02f1b7e6 2024-11-13T19:25:00,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/f294b92fc5ba478097821561f4a3b5b5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/f294b92fc5ba478097821561f4a3b5b5 2024-11-13T19:25:00,671 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/f294b92fc5ba478097821561f4a3b5b5, entries=150, sequenceid=166, filesize=30.4 K 2024-11-13T19:25:00,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/67710ec385a9404e8729eb3b5a0d5dd5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/67710ec385a9404e8729eb3b5a0d5dd5 2024-11-13T19:25:00,674 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/67710ec385a9404e8729eb3b5a0d5dd5, entries=150, sequenceid=166, filesize=11.9 K 2024-11-13T19:25:00,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0df4a2ae32d04bf8b358b62e02f1b7e6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0df4a2ae32d04bf8b358b62e02f1b7e6 2024-11-13T19:25:00,678 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0df4a2ae32d04bf8b358b62e02f1b7e6, entries=150, sequenceid=166, filesize=11.9 K 2024-11-13T19:25:00,679 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for b04e7477f4d6e72d4397199d014ea80b in 1654ms, sequenceid=166, compaction requested=false 2024-11-13T19:25:00,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:00,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:00,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-13T19:25:00,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-13T19:25:00,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-13T19:25:00,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3360 sec 2024-11-13T19:25:00,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.3390 sec 2024-11-13T19:25:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:00,739 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:00,739 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-13T19:25:00,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113898b8daddc614562abe1a3515c1964ce_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525899561/Put/seqid=0 2024-11-13T19:25:00,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742189_1365 (size=14794) 2024-11-13T19:25:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525960756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:00,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:00,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525960757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525960871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525960871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:00,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525961082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:01,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525961082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:01,150 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:01,153 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113898b8daddc614562abe1a3515c1964ce_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113898b8daddc614562abe1a3515c1964ce_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:01,154 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/bf5ef0d474774b5b966253f19c94dcbc, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:01,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/bf5ef0d474774b5b966253f19c94dcbc is 175, key is test_row_0/A:col10/1731525899561/Put/seqid=0 2024-11-13T19:25:01,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742190_1366 (size=39749) 2024-11-13T19:25:01,158 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/bf5ef0d474774b5b966253f19c94dcbc 2024-11-13T19:25:01,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/4703d779394846a5a1dfe7ee377098ec is 50, key is test_row_0/B:col10/1731525899561/Put/seqid=0 2024-11-13T19:25:01,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742191_1367 (size=12151) 2024-11-13T19:25:01,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525961391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:01,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525961391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,397 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:01,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-13T19:25:01,501 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:01,502 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:01,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-13T19:25:01,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:25:01,503 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:01,504 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:01,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:01,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/4703d779394846a5a1dfe7ee377098ec 2024-11-13T19:25:01,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/6477c6fa157a4db6aa37407bcf7f3561 is 50, key is test_row_0/C:col10/1731525899561/Put/seqid=0 2024-11-13T19:25:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742192_1368 (size=12151) 2024-11-13T19:25:01,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:25:01,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T19:25:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:01,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T19:25:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:01,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:25:01,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:01,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525961901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:01,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525961901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:01,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T19:25:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:01,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:01,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/6477c6fa157a4db6aa37407bcf7f3561 2024-11-13T19:25:01,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/bf5ef0d474774b5b966253f19c94dcbc as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/bf5ef0d474774b5b966253f19c94dcbc 2024-11-13T19:25:01,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/bf5ef0d474774b5b966253f19c94dcbc, entries=200, sequenceid=196, filesize=38.8 K 2024-11-13T19:25:01,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/4703d779394846a5a1dfe7ee377098ec as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/4703d779394846a5a1dfe7ee377098ec 2024-11-13T19:25:01,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/4703d779394846a5a1dfe7ee377098ec, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T19:25:01,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/6477c6fa157a4db6aa37407bcf7f3561 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6477c6fa157a4db6aa37407bcf7f3561 2024-11-13T19:25:02,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6477c6fa157a4db6aa37407bcf7f3561, entries=150, sequenceid=196, filesize=11.9 K 2024-11-13T19:25:02,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for b04e7477f4d6e72d4397199d014ea80b in 1263ms, sequenceid=196, compaction requested=true 2024-11-13T19:25:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:02,002 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:02,002 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:02,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:02,003 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:02,003 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:02,003 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/B is initiating minor compaction (all files) 2024-11-13T19:25:02,003 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/A is initiating minor compaction (all files) 2024-11-13T19:25:02,003 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/B in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:02,003 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/A in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:02,003 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ff013bc5d0a345d2a16f7547defcee6b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/67710ec385a9404e8729eb3b5a0d5dd5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/4703d779394846a5a1dfe7ee377098ec] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=35.9 K 2024-11-13T19:25:02,003 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0367befe73b34c4599c47328b0774e34, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/f294b92fc5ba478097821561f4a3b5b5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/bf5ef0d474774b5b966253f19c94dcbc] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=99.9 K 2024-11-13T19:25:02,003 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:02,003 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0367befe73b34c4599c47328b0774e34, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/f294b92fc5ba478097821561f4a3b5b5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/bf5ef0d474774b5b966253f19c94dcbc] 2024-11-13T19:25:02,004 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting ff013bc5d0a345d2a16f7547defcee6b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525896917 2024-11-13T19:25:02,004 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0367befe73b34c4599c47328b0774e34, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525896917 2024-11-13T19:25:02,004 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 67710ec385a9404e8729eb3b5a0d5dd5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731525897328 2024-11-13T19:25:02,004 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting f294b92fc5ba478097821561f4a3b5b5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731525897328 2024-11-13T19:25:02,004 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4703d779394846a5a1dfe7ee377098ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731525899561 2024-11-13T19:25:02,004 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf5ef0d474774b5b966253f19c94dcbc, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731525899561 2024-11-13T19:25:02,015 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:02,017 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#B#compaction#312 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:02,017 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/a8e376d231754a50af0580db77111f69 is 50, key is test_row_0/B:col10/1731525899561/Put/seqid=0 2024-11-13T19:25:02,026 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111369aa7097f01342afa5ef94d6fb4fa49a_b04e7477f4d6e72d4397199d014ea80b store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:02,028 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111369aa7097f01342afa5ef94d6fb4fa49a_b04e7477f4d6e72d4397199d014ea80b, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:02,028 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111369aa7097f01342afa5ef94d6fb4fa49a_b04e7477f4d6e72d4397199d014ea80b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:02,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742193_1369 (size=12595) 2024-11-13T19:25:02,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742194_1370 (size=4469) 2024-11-13T19:25:02,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-13T19:25:02,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:02,115 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-13T19:25:02,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4e0087944f54040a16f28876155f70b_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525900741/Put/seqid=0 2024-11-13T19:25:02,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742195_1371 (size=12304) 2024-11-13T19:25:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:25:02,434 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/a8e376d231754a50af0580db77111f69 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a8e376d231754a50af0580db77111f69 2024-11-13T19:25:02,439 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/B of b04e7477f4d6e72d4397199d014ea80b into a8e376d231754a50af0580db77111f69(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:02,439 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:02,439 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/B, priority=13, startTime=1731525902002; duration=0sec 2024-11-13T19:25:02,439 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:02,439 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:B 2024-11-13T19:25:02,439 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:02,439 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#A#compaction#313 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:02,440 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/b5216390dbe34b61bbd0e15ba442896f is 175, key is test_row_0/A:col10/1731525899561/Put/seqid=0 2024-11-13T19:25:02,440 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:02,440 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/C is initiating minor compaction (all files) 2024-11-13T19:25:02,440 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/C in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:02,440 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f766ad121f7e4e68bae8193030efa1b7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0df4a2ae32d04bf8b358b62e02f1b7e6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6477c6fa157a4db6aa37407bcf7f3561] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=35.9 K 2024-11-13T19:25:02,441 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f766ad121f7e4e68bae8193030efa1b7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731525896917 2024-11-13T19:25:02,441 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0df4a2ae32d04bf8b358b62e02f1b7e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731525897328 2024-11-13T19:25:02,441 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6477c6fa157a4db6aa37407bcf7f3561, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731525899561 2024-11-13T19:25:02,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742196_1372 (size=31549) 2024-11-13T19:25:02,448 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:02,449 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/3dbd3bebdbdd4d329652c5622816c327 is 50, key is test_row_0/C:col10/1731525899561/Put/seqid=0 2024-11-13T19:25:02,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742197_1373 (size=12595) 2024-11-13T19:25:02,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:02,528 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113a4e0087944f54040a16f28876155f70b_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4e0087944f54040a16f28876155f70b_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0caec87dff184530b6b4e4a1dcf2b2dc, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:02,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0caec87dff184530b6b4e4a1dcf2b2dc is 175, key is test_row_0/A:col10/1731525900741/Put/seqid=0 2024-11-13T19:25:02,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742198_1374 (size=31105) 2024-11-13T19:25:02,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:25:02,847 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/b5216390dbe34b61bbd0e15ba442896f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b5216390dbe34b61bbd0e15ba442896f 2024-11-13T19:25:02,851 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/A of b04e7477f4d6e72d4397199d014ea80b into b5216390dbe34b61bbd0e15ba442896f(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:02,851 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:02,851 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/A, priority=13, startTime=1731525902002; duration=0sec 2024-11-13T19:25:02,851 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:02,851 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:A 2024-11-13T19:25:02,856 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/3dbd3bebdbdd4d329652c5622816c327 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3dbd3bebdbdd4d329652c5622816c327 2024-11-13T19:25:02,860 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/C of b04e7477f4d6e72d4397199d014ea80b into 3dbd3bebdbdd4d329652c5622816c327(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:02,860 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:02,860 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/C, priority=13, startTime=1731525902002; duration=0sec 2024-11-13T19:25:02,860 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:02,860 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:C 2024-11-13T19:25:02,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:02,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:02,935 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0caec87dff184530b6b4e4a1dcf2b2dc 2024-11-13T19:25:02,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/91d9fd95f5be4cf185d94a8de72129fb is 50, key is test_row_0/B:col10/1731525900741/Put/seqid=0 2024-11-13T19:25:02,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742199_1375 (size=12151) 2024-11-13T19:25:02,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:02,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525962989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:02,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:02,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:02,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:02,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525962990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:02,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525963101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525963101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,108 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525963311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525963311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,348 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/91d9fd95f5be4cf185d94a8de72129fb 2024-11-13T19:25:03,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0cf563a2b0e14a3fa53c88d99202b9a2 is 50, key is test_row_0/C:col10/1731525900741/Put/seqid=0 2024-11-13T19:25:03,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742200_1376 (size=12151) 2024-11-13T19:25:03,366 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0cf563a2b0e14a3fa53c88d99202b9a2 2024-11-13T19:25:03,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0caec87dff184530b6b4e4a1dcf2b2dc as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0caec87dff184530b6b4e4a1dcf2b2dc 2024-11-13T19:25:03,378 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0caec87dff184530b6b4e4a1dcf2b2dc, entries=150, sequenceid=205, filesize=30.4 K 2024-11-13T19:25:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/91d9fd95f5be4cf185d94a8de72129fb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/91d9fd95f5be4cf185d94a8de72129fb 2024-11-13T19:25:03,383 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/91d9fd95f5be4cf185d94a8de72129fb, entries=150, sequenceid=205, filesize=11.9 K 2024-11-13T19:25:03,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0cf563a2b0e14a3fa53c88d99202b9a2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0cf563a2b0e14a3fa53c88d99202b9a2 2024-11-13T19:25:03,390 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0cf563a2b0e14a3fa53c88d99202b9a2, entries=150, sequenceid=205, filesize=11.9 K 2024-11-13T19:25:03,391 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for b04e7477f4d6e72d4397199d014ea80b in 1276ms, sequenceid=205, compaction requested=false 2024-11-13T19:25:03,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:03,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:03,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-13T19:25:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-13T19:25:03,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-13T19:25:03,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8890 sec 2024-11-13T19:25:03,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8960 sec 2024-11-13T19:25:03,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:03,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:03,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-13T19:25:03,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111356d0e4b9eb604500a4f80857933ce08a_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525902977/Put/seqid=0 2024-11-13T19:25:03,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742201_1377 (size=14794) 2024-11-13T19:25:03,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:03,644 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111356d0e4b9eb604500a4f80857933ce08a_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111356d0e4b9eb604500a4f80857933ce08a_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:03,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/65661bbdef4d4fef89d6015d9ad2cfe5, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:03,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/65661bbdef4d4fef89d6015d9ad2cfe5 is 175, key is test_row_0/A:col10/1731525902977/Put/seqid=0 2024-11-13T19:25:03,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742202_1378 (size=39749) 2024-11-13T19:25:03,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-13T19:25:03,651 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:03,652 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-13T19:25:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T19:25:03,653 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:03,654 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:03,654 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:03,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525963646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525963647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T19:25:03,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525963761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525963761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:03,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:03,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:03,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:03,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:03,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:03,958 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:03,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:03,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T19:25:03,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525963971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525963971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:03,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:04,049 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/65661bbdef4d4fef89d6015d9ad2cfe5 2024-11-13T19:25:04,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/dfab96578ad04e4098976ea4aa92c855 is 50, key is test_row_0/B:col10/1731525902977/Put/seqid=0 2024-11-13T19:25:04,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742203_1379 (size=12151) 2024-11-13T19:25:04,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,264 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T19:25:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525964282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:04,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525964282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:04,417 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:04,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:04,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,417 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/dfab96578ad04e4098976ea4aa92c855 2024-11-13T19:25:04,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a is 50, key is test_row_0/C:col10/1731525902977/Put/seqid=0 2024-11-13T19:25:04,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742204_1380 (size=12151) 2024-11-13T19:25:04,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:04,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T19:25:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525964791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:04,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525964791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:04,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a 2024-11-13T19:25:04,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/65661bbdef4d4fef89d6015d9ad2cfe5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/65661bbdef4d4fef89d6015d9ad2cfe5 2024-11-13T19:25:04,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/65661bbdef4d4fef89d6015d9ad2cfe5, entries=200, sequenceid=236, filesize=38.8 K 2024-11-13T19:25:04,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/dfab96578ad04e4098976ea4aa92c855 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/dfab96578ad04e4098976ea4aa92c855 2024-11-13T19:25:04,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:04,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/dfab96578ad04e4098976ea4aa92c855, entries=150, sequenceid=236, filesize=11.9 K 2024-11-13T19:25:04,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a 2024-11-13T19:25:04,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a, entries=150, sequenceid=236, filesize=11.9 K 2024-11-13T19:25:04,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for b04e7477f4d6e72d4397199d014ea80b in 1259ms, sequenceid=236, compaction requested=true 2024-11-13T19:25:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:04,887 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:04,887 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:04,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:04,888 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:04,888 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/B is initiating minor compaction (all files) 2024-11-13T19:25:04,888 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/B in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,888 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:04,888 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/A is initiating minor compaction (all files) 2024-11-13T19:25:04,888 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a8e376d231754a50af0580db77111f69, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/91d9fd95f5be4cf185d94a8de72129fb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/dfab96578ad04e4098976ea4aa92c855] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=36.0 K 2024-11-13T19:25:04,888 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/A in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,888 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b5216390dbe34b61bbd0e15ba442896f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0caec87dff184530b6b4e4a1dcf2b2dc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/65661bbdef4d4fef89d6015d9ad2cfe5] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=100.0 K 2024-11-13T19:25:04,888 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:04,888 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b5216390dbe34b61bbd0e15ba442896f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0caec87dff184530b6b4e4a1dcf2b2dc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/65661bbdef4d4fef89d6015d9ad2cfe5] 2024-11-13T19:25:04,889 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a8e376d231754a50af0580db77111f69, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731525899561 2024-11-13T19:25:04,889 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5216390dbe34b61bbd0e15ba442896f, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731525899561 2024-11-13T19:25:04,889 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0caec87dff184530b6b4e4a1dcf2b2dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731525900741 2024-11-13T19:25:04,889 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 91d9fd95f5be4cf185d94a8de72129fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731525900741 2024-11-13T19:25:04,890 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65661bbdef4d4fef89d6015d9ad2cfe5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731525902977 2024-11-13T19:25:04,890 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting dfab96578ad04e4098976ea4aa92c855, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731525902977 2024-11-13T19:25:04,895 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:04,898 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:04,898 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ccc6c4f3f89a495c99cd062be17b5de8 is 50, key is test_row_0/B:col10/1731525902977/Put/seqid=0 2024-11-13T19:25:04,899 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411132c49680ccac8457f9cbea6ccd7511fbf_b04e7477f4d6e72d4397199d014ea80b store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:04,900 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411132c49680ccac8457f9cbea6ccd7511fbf_b04e7477f4d6e72d4397199d014ea80b, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:04,900 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411132c49680ccac8457f9cbea6ccd7511fbf_b04e7477f4d6e72d4397199d014ea80b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:04,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742205_1381 (size=4469) 2024-11-13T19:25:04,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742206_1382 (size=12697) 2024-11-13T19:25:05,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-13T19:25:05,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:05,030 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T19:25:05,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131483aeb21d22409f835548bb34cf8170_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525903645/Put/seqid=0 2024-11-13T19:25:05,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742207_1383 (size=12304) 2024-11-13T19:25:05,305 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#A#compaction#321 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:05,305 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/d296c09e660745198a079a1eaeeecd1a is 175, key is test_row_0/A:col10/1731525902977/Put/seqid=0 2024-11-13T19:25:05,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742208_1384 (size=31651) 2024-11-13T19:25:05,321 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ccc6c4f3f89a495c99cd062be17b5de8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ccc6c4f3f89a495c99cd062be17b5de8 2024-11-13T19:25:05,325 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/B of b04e7477f4d6e72d4397199d014ea80b into ccc6c4f3f89a495c99cd062be17b5de8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:05,325 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:05,325 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/B, priority=13, startTime=1731525904887; duration=0sec 2024-11-13T19:25:05,325 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:05,325 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:B 2024-11-13T19:25:05,325 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:05,326 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:05,326 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/C is initiating minor compaction (all files) 2024-11-13T19:25:05,326 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/C in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:05,326 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3dbd3bebdbdd4d329652c5622816c327, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0cf563a2b0e14a3fa53c88d99202b9a2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=36.0 K 2024-11-13T19:25:05,326 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3dbd3bebdbdd4d329652c5622816c327, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731525899561 2024-11-13T19:25:05,327 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cf563a2b0e14a3fa53c88d99202b9a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731525900741 2024-11-13T19:25:05,327 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd5cacf4dc74fdfa1cf7b1d6f412d5a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731525902977 2024-11-13T19:25:05,333 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#C#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:05,334 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/4b66334ccd46467280a2819338e037d8 is 50, key is test_row_0/C:col10/1731525902977/Put/seqid=0 2024-11-13T19:25:05,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742209_1385 (size=12697) 2024-11-13T19:25:05,344 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/4b66334ccd46467280a2819338e037d8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4b66334ccd46467280a2819338e037d8 2024-11-13T19:25:05,350 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/C of b04e7477f4d6e72d4397199d014ea80b into 4b66334ccd46467280a2819338e037d8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:05,350 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:05,350 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/C, priority=13, startTime=1731525904887; duration=0sec 2024-11-13T19:25:05,350 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:05,350 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:C 2024-11-13T19:25:05,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:05,455 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131483aeb21d22409f835548bb34cf8170_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131483aeb21d22409f835548bb34cf8170_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:05,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/b398d010ea1a4693bc7678e0fd5a65d2, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:05,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/b398d010ea1a4693bc7678e0fd5a65d2 is 175, key is test_row_0/A:col10/1731525903645/Put/seqid=0 2024-11-13T19:25:05,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742210_1386 (size=31105) 2024-11-13T19:25:05,748 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/d296c09e660745198a079a1eaeeecd1a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/d296c09e660745198a079a1eaeeecd1a 2024-11-13T19:25:05,752 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/A of b04e7477f4d6e72d4397199d014ea80b into d296c09e660745198a079a1eaeeecd1a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:05,752 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:05,752 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/A, priority=13, startTime=1731525904887; duration=0sec 2024-11-13T19:25:05,752 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:05,752 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:A 2024-11-13T19:25:05,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T19:25:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:05,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:05,861 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/b398d010ea1a4693bc7678e0fd5a65d2 2024-11-13T19:25:05,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/51e53b5ea6924cc0b938f4aae6a1ae24 is 50, key is test_row_0/B:col10/1731525903645/Put/seqid=0 2024-11-13T19:25:05,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742211_1387 (size=12151) 2024-11-13T19:25:05,876 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/51e53b5ea6924cc0b938f4aae6a1ae24 2024-11-13T19:25:05,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0f53c91755b949f1b9fc285de62a2178 is 50, key is test_row_0/C:col10/1731525903645/Put/seqid=0 2024-11-13T19:25:05,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742212_1388 (size=12151) 2024-11-13T19:25:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525965881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:05,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:05,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525965885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525966001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525966001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525966212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525966212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,289 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0f53c91755b949f1b9fc285de62a2178 2024-11-13T19:25:06,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/b398d010ea1a4693bc7678e0fd5a65d2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b398d010ea1a4693bc7678e0fd5a65d2 2024-11-13T19:25:06,296 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b398d010ea1a4693bc7678e0fd5a65d2, entries=150, sequenceid=244, filesize=30.4 K 2024-11-13T19:25:06,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/51e53b5ea6924cc0b938f4aae6a1ae24 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/51e53b5ea6924cc0b938f4aae6a1ae24 2024-11-13T19:25:06,299 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/51e53b5ea6924cc0b938f4aae6a1ae24, entries=150, sequenceid=244, filesize=11.9 K 2024-11-13T19:25:06,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0f53c91755b949f1b9fc285de62a2178 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0f53c91755b949f1b9fc285de62a2178 2024-11-13T19:25:06,303 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0f53c91755b949f1b9fc285de62a2178, entries=150, sequenceid=244, filesize=11.9 K 2024-11-13T19:25:06,304 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for b04e7477f4d6e72d4397199d014ea80b in 1274ms, sequenceid=244, compaction requested=false 2024-11-13T19:25:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-13T19:25:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-13T19:25:06,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-13T19:25:06,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6520 sec 2024-11-13T19:25:06,310 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.6570 sec 2024-11-13T19:25:06,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:06,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:06,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-13T19:25:06,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b9e5362c7b3e4960bbaa8391445ca472_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525906524/Put/seqid=0 2024-11-13T19:25:06,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742213_1389 (size=14994) 2024-11-13T19:25:06,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525966547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525966548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525966651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525966662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525966861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525966871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:06,940 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:06,943 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113b9e5362c7b3e4960bbaa8391445ca472_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b9e5362c7b3e4960bbaa8391445ca472_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:06,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/15c6caf29ec243a39b9e5993cac62f00, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:06,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/15c6caf29ec243a39b9e5993cac62f00 is 175, key is test_row_0/A:col10/1731525906524/Put/seqid=0 2024-11-13T19:25:06,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742214_1390 (size=39949) 2024-11-13T19:25:06,948 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/15c6caf29ec243a39b9e5993cac62f00 2024-11-13T19:25:06,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/823d7c654a7f4243ac63f1c6e80ae583 is 50, key is test_row_0/B:col10/1731525906524/Put/seqid=0 2024-11-13T19:25:06,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742215_1391 (size=12301) 2024-11-13T19:25:07,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525967172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,177 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:07,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:07,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525967181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:07,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/823d7c654a7f4243ac63f1c6e80ae583 2024-11-13T19:25:07,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0ccc143b65bc42f6a7b3ef42a793cbc5 is 50, key is test_row_0/C:col10/1731525906524/Put/seqid=0 2024-11-13T19:25:07,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742216_1392 (size=12301) 2024-11-13T19:25:07,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:07,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525967682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:07,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:07,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525967701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:07,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:07,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0ccc143b65bc42f6a7b3ef42a793cbc5 2024-11-13T19:25:07,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/15c6caf29ec243a39b9e5993cac62f00 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/15c6caf29ec243a39b9e5993cac62f00 2024-11-13T19:25:07,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/15c6caf29ec243a39b9e5993cac62f00, entries=200, sequenceid=276, filesize=39.0 K 2024-11-13T19:25:07,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/823d7c654a7f4243ac63f1c6e80ae583 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/823d7c654a7f4243ac63f1c6e80ae583 2024-11-13T19:25:07,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/823d7c654a7f4243ac63f1c6e80ae583, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T19:25:07,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/0ccc143b65bc42f6a7b3ef42a793cbc5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0ccc143b65bc42f6a7b3ef42a793cbc5 2024-11-13T19:25:07,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0ccc143b65bc42f6a7b3ef42a793cbc5, entries=150, sequenceid=276, filesize=12.0 K 2024-11-13T19:25:07,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for b04e7477f4d6e72d4397199d014ea80b in 1257ms, sequenceid=276, compaction requested=true 2024-11-13T19:25:07,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:07,783 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:07,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:07,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:07,783 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:07,784 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:07,784 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:07,785 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/A is initiating minor compaction (all files) 2024-11-13T19:25:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:07,785 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:07,785 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/A in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:07,785 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/d296c09e660745198a079a1eaeeecd1a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b398d010ea1a4693bc7678e0fd5a65d2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/15c6caf29ec243a39b9e5993cac62f00] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=100.3 K 2024-11-13T19:25:07,785 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:07,785 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/d296c09e660745198a079a1eaeeecd1a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b398d010ea1a4693bc7678e0fd5a65d2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/15c6caf29ec243a39b9e5993cac62f00] 2024-11-13T19:25:07,785 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:07,785 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/B is initiating minor compaction (all files) 2024-11-13T19:25:07,785 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d296c09e660745198a079a1eaeeecd1a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731525902977 2024-11-13T19:25:07,785 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/B in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:07,785 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ccc6c4f3f89a495c99cd062be17b5de8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/51e53b5ea6924cc0b938f4aae6a1ae24, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/823d7c654a7f4243ac63f1c6e80ae583] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=36.3 K 2024-11-13T19:25:07,786 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b398d010ea1a4693bc7678e0fd5a65d2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731525903637 2024-11-13T19:25:07,786 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting ccc6c4f3f89a495c99cd062be17b5de8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731525902977 2024-11-13T19:25:07,786 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 51e53b5ea6924cc0b938f4aae6a1ae24, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731525903637 2024-11-13T19:25:07,786 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c6caf29ec243a39b9e5993cac62f00, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731525905873 2024-11-13T19:25:07,786 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 823d7c654a7f4243ac63f1c6e80ae583, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731525905873 2024-11-13T19:25:07,798 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:07,799 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#B#compaction#330 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:07,800 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/1194ba540a2e45548b5977a5bb895041 is 50, key is test_row_0/B:col10/1731525906524/Put/seqid=0 2024-11-13T19:25:07,803 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411133dabd47cde7c48a9ae83d19bd84174eb_b04e7477f4d6e72d4397199d014ea80b store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:07,805 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411133dabd47cde7c48a9ae83d19bd84174eb_b04e7477f4d6e72d4397199d014ea80b, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:07,805 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133dabd47cde7c48a9ae83d19bd84174eb_b04e7477f4d6e72d4397199d014ea80b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:07,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742217_1393 (size=12949) 2024-11-13T19:25:07,826 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/1194ba540a2e45548b5977a5bb895041 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/1194ba540a2e45548b5977a5bb895041 2024-11-13T19:25:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-13T19:25:07,832 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:07,832 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:07,832 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/B of b04e7477f4d6e72d4397199d014ea80b into 1194ba540a2e45548b5977a5bb895041(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:07,832 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:07,833 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/B, priority=13, startTime=1731525907783; duration=0sec 2024-11-13T19:25:07,833 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:07,833 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:B 2024-11-13T19:25:07,833 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:07,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-13T19:25:07,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:25:07,836 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:07,836 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): b04e7477f4d6e72d4397199d014ea80b/C is initiating minor compaction (all files) 2024-11-13T19:25:07,836 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:07,836 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b04e7477f4d6e72d4397199d014ea80b/C in TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:07,836 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4b66334ccd46467280a2819338e037d8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0f53c91755b949f1b9fc285de62a2178, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0ccc143b65bc42f6a7b3ef42a793cbc5] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp, totalSize=36.3 K 2024-11-13T19:25:07,837 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b66334ccd46467280a2819338e037d8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731525902977 2024-11-13T19:25:07,837 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:07,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:07,838 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f53c91755b949f1b9fc285de62a2178, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731525903637 2024-11-13T19:25:07,838 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ccc143b65bc42f6a7b3ef42a793cbc5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731525905873 2024-11-13T19:25:07,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742218_1394 (size=4469) 2024-11-13T19:25:07,848 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#C#compaction#332 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:07,849 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/de309e47b3ec4918af5f9c12892c404a is 50, key is test_row_0/C:col10/1731525906524/Put/seqid=0 2024-11-13T19:25:07,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742219_1395 (size=12949) 2024-11-13T19:25:07,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:25:07,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-13T19:25:07,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:07,991 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T19:25:07,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e19627aba34e4abdae9cd03b81aa8258_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525906546/Put/seqid=0 2024-11-13T19:25:07,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742220_1396 (size=12454) 2024-11-13T19:25:08,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:25:08,246 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b04e7477f4d6e72d4397199d014ea80b#A#compaction#331 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:08,246 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0c36c3f180bb42b09dde4e622aa21b0d is 175, key is test_row_0/A:col10/1731525906524/Put/seqid=0 2024-11-13T19:25:08,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742221_1397 (size=31903) 2024-11-13T19:25:08,266 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/de309e47b3ec4918af5f9c12892c404a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/de309e47b3ec4918af5f9c12892c404a 2024-11-13T19:25:08,270 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/C of b04e7477f4d6e72d4397199d014ea80b into de309e47b3ec4918af5f9c12892c404a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:08,270 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:08,270 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/C, priority=13, startTime=1731525907785; duration=0sec 2024-11-13T19:25:08,270 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:08,270 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:C 2024-11-13T19:25:08,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:08,403 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113e19627aba34e4abdae9cd03b81aa8258_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e19627aba34e4abdae9cd03b81aa8258_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:08,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/9a99873395d04510afca86cebf442443, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:08,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/9a99873395d04510afca86cebf442443 is 175, key is test_row_0/A:col10/1731525906546/Put/seqid=0 2024-11-13T19:25:08,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742222_1398 (size=31255) 2024-11-13T19:25:08,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:08,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. as already flushing 2024-11-13T19:25:08,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:25:08,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525968535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525968536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525968537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,653 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/0c36c3f180bb42b09dde4e622aa21b0d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0c36c3f180bb42b09dde4e622aa21b0d 2024-11-13T19:25:08,657 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b04e7477f4d6e72d4397199d014ea80b/A of b04e7477f4d6e72d4397199d014ea80b into 0c36c3f180bb42b09dde4e622aa21b0d(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:08,657 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:08,657 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., storeName=b04e7477f4d6e72d4397199d014ea80b/A, priority=13, startTime=1731525907783; duration=0sec 2024-11-13T19:25:08,657 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:08,657 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:A 2024-11-13T19:25:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525968651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525968651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525968651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57180 deadline: 1731525968701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,703 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57194 deadline: 1731525968721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,728 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in b04e7477f4d6e72d4397199d014ea80b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,734 INFO [Thread-1470 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1470. 2024-11-13T19:25:08,734 DEBUG [Thread-1470 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:08,734 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,734 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-11-13T19:25:08,734 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:08,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:08,734 DEBUG [Thread-1470 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,734 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,734 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,734 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:08,740 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-13T19:25:08,740 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:08,740 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,741 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:08,741 INFO [Thread-1464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1464. 2024-11-13T19:25:08,741 DEBUG [Thread-1464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:08,741 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,741 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,741 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:08,742 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-11-13T19:25:08,742 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:08,742 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,743 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:08,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:08,810 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/9a99873395d04510afca86cebf442443 2024-11-13T19:25:08,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/e0cb77c16b4c4dafbb8bd4ff92a6a2bb is 50, key is test_row_0/B:col10/1731525906546/Put/seqid=0 2024-11-13T19:25:08,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742223_1399 (size=12301) 2024-11-13T19:25:08,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525968862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525968862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525968862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:25:09,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:09,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:09,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57240 deadline: 1731525969172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57248 deadline: 1731525969172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:09,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57228 deadline: 1731525969172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:09,227 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/e0cb77c16b4c4dafbb8bd4ff92a6a2bb 2024-11-13T19:25:09,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/512f7dc7cb224b519599f0d4deef8620 is 50, key is test_row_0/C:col10/1731525906546/Put/seqid=0 2024-11-13T19:25:09,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742224_1400 (size=12301) 2024-11-13T19:25:09,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/512f7dc7cb224b519599f0d4deef8620 2024-11-13T19:25:09,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/9a99873395d04510afca86cebf442443 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/9a99873395d04510afca86cebf442443 2024-11-13T19:25:09,652 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/9a99873395d04510afca86cebf442443, entries=150, sequenceid=285, filesize=30.5 K 2024-11-13T19:25:09,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/e0cb77c16b4c4dafbb8bd4ff92a6a2bb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0cb77c16b4c4dafbb8bd4ff92a6a2bb 2024-11-13T19:25:09,657 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0cb77c16b4c4dafbb8bd4ff92a6a2bb, entries=150, sequenceid=285, filesize=12.0 K 2024-11-13T19:25:09,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/512f7dc7cb224b519599f0d4deef8620 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/512f7dc7cb224b519599f0d4deef8620 2024-11-13T19:25:09,661 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/512f7dc7cb224b519599f0d4deef8620, entries=150, sequenceid=285, filesize=12.0 K 2024-11-13T19:25:09,662 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for b04e7477f4d6e72d4397199d014ea80b in 1671ms, sequenceid=285, compaction requested=false 2024-11-13T19:25:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:09,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-13T19:25:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-13T19:25:09,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-13T19:25:09,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8260 sec 2024-11-13T19:25:09,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.8330 sec 2024-11-13T19:25:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:09,683 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:09,684 INFO [Thread-1457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1457. 2024-11-13T19:25:09,684 INFO [Thread-1451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1451. 2024-11-13T19:25:09,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=187.85 KB heapSize=492.94 KB 2024-11-13T19:25:09,684 DEBUG [Thread-1451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:09,684 DEBUG [Thread-1457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:09,684 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:09,684 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:09,684 INFO [Thread-1455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1455. 2024-11-13T19:25:09,684 DEBUG [Thread-1455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:09,684 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:09,684 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:09,684 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:09,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:09,684 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:09,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:09,684 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:09,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc90c0dfbeb74d95a80a7cd55494b65a_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525909682/Put/seqid=0 2024-11-13T19:25:09,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742225_1401 (size=12454) 2024-11-13T19:25:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-13T19:25:09,994 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:10,095 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:10,104 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113dc90c0dfbeb74d95a80a7cd55494b65a_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc90c0dfbeb74d95a80a7cd55494b65a_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:10,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/2221442175754ca99486545ec557f41e, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:10,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/2221442175754ca99486545ec557f41e is 175, key is test_row_0/A:col10/1731525909682/Put/seqid=0 2024-11-13T19:25:10,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742226_1402 (size=31255) 2024-11-13T19:25:10,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=62.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/2221442175754ca99486545ec557f41e 2024-11-13T19:25:10,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/e0320143d5f846f0baaf4b4fbc548789 is 50, key is test_row_0/B:col10/1731525909682/Put/seqid=0 2024-11-13T19:25:10,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742227_1403 (size=12301) 2024-11-13T19:25:10,717 INFO [Thread-1459 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1459. 2024-11-13T19:25:10,717 DEBUG [Thread-1459 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:10,717 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:10,717 DEBUG [Thread-1459 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:10,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:10,752 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:25:10,762 INFO [Thread-1453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1453. 2024-11-13T19:25:10,762 DEBUG [Thread-1453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:10,762 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:10,762 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:10,762 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1638 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4909 rows 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1639 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4915 rows 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1642 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4924 rows 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1640 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4919 rows 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1636 2024-11-13T19:25:10,763 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4907 rows 2024-11-13T19:25:10,763 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:25:10,763 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:25:10,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:10,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:10,764 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:10,764 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:25:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:25:10,768 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525910768"}]},"ts":"1731525910768"} 2024-11-13T19:25:10,770 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:25:10,770 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:25:10,770 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:25:10,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, UNASSIGN}] 2024-11-13T19:25:10,772 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, UNASSIGN 2024-11-13T19:25:10,772 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:10,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, UNASSIGN because future has completed 2024-11-13T19:25:10,774 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:25:10,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:25:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:25:10,926 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:10,926 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:25:10,926 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing b04e7477f4d6e72d4397199d014ea80b, disabling compactions & flushes 2024-11-13T19:25:10,926 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:10,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/e0320143d5f846f0baaf4b4fbc548789 2024-11-13T19:25:10,946 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/1efc2d46927c4aa3b9053c69ad82c9b8 is 50, key is test_row_0/C:col10/1731525909682/Put/seqid=0 2024-11-13T19:25:10,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742228_1404 (size=12301) 2024-11-13T19:25:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:25:11,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/1efc2d46927c4aa3b9053c69ad82c9b8 2024-11-13T19:25:11,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/2221442175754ca99486545ec557f41e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/2221442175754ca99486545ec557f41e 2024-11-13T19:25:11,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/2221442175754ca99486545ec557f41e, entries=150, sequenceid=318, filesize=30.5 K 2024-11-13T19:25:11,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/e0320143d5f846f0baaf4b4fbc548789 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0320143d5f846f0baaf4b4fbc548789 2024-11-13T19:25:11,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0320143d5f846f0baaf4b4fbc548789, entries=150, sequenceid=318, filesize=12.0 K 2024-11-13T19:25:11,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/1efc2d46927c4aa3b9053c69ad82c9b8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/1efc2d46927c4aa3b9053c69ad82c9b8 2024-11-13T19:25:11,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/1efc2d46927c4aa3b9053c69ad82c9b8, entries=150, sequenceid=318, filesize=12.0 K 2024-11-13T19:25:11,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~187.85 KB/192360, heapSize ~492.89 KB/504720, currentSize=13.42 KB/13740 for b04e7477f4d6e72d4397199d014ea80b in 1699ms, sequenceid=318, compaction requested=true 2024-11-13T19:25:11,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b04e7477f4d6e72d4397199d014ea80b: 2024-11-13T19:25:11,382 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:11,383 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:11,383 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. after waiting 0 ms 2024-11-13T19:25:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:11,383 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b04e7477f4d6e72d4397199d014ea80b:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:11,383 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing b04e7477f4d6e72d4397199d014ea80b 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-13T19:25:11,383 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. because compaction request was cancelled 2024-11-13T19:25:11,383 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. because compaction request was cancelled 2024-11-13T19:25:11,383 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:A 2024-11-13T19:25:11,383 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:B 2024-11-13T19:25:11,383 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. because compaction request was cancelled 2024-11-13T19:25:11,383 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b04e7477f4d6e72d4397199d014ea80b:C 2024-11-13T19:25:11,390 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113741b24aa620a47cda4f284eb83c881cc_b04e7477f4d6e72d4397199d014ea80b is 50, key is test_row_0/A:col10/1731525910752/Put/seqid=0 2024-11-13T19:25:11,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:25:11,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742229_1405 (size=7374) 2024-11-13T19:25:11,796 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:11,806 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113741b24aa620a47cda4f284eb83c881cc_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113741b24aa620a47cda4f284eb83c881cc_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:11,809 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/ee2e09e61eac470b8ba4e0c48f7e20d5, store: [table=TestAcidGuarantees family=A region=b04e7477f4d6e72d4397199d014ea80b] 2024-11-13T19:25:11,811 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/ee2e09e61eac470b8ba4e0c48f7e20d5 is 175, key is test_row_0/A:col10/1731525910752/Put/seqid=0 2024-11-13T19:25:11,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742230_1406 (size=13865) 2024-11-13T19:25:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:25:12,218 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=323, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/ee2e09e61eac470b8ba4e0c48f7e20d5 2024-11-13T19:25:12,230 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ea88a4038f3a46ee8f0bf3a7b73b2214 is 50, key is test_row_0/B:col10/1731525910752/Put/seqid=0 2024-11-13T19:25:12,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742231_1407 (size=7415) 2024-11-13T19:25:12,635 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ea88a4038f3a46ee8f0bf3a7b73b2214 2024-11-13T19:25:12,647 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/77e6841cbef14d6f85687aad0305cece is 50, key is test_row_0/C:col10/1731525910752/Put/seqid=0 2024-11-13T19:25:12,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742232_1408 (size=7415) 2024-11-13T19:25:12,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:25:13,052 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=323 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/77e6841cbef14d6f85687aad0305cece 2024-11-13T19:25:13,065 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/A/ee2e09e61eac470b8ba4e0c48f7e20d5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/ee2e09e61eac470b8ba4e0c48f7e20d5 2024-11-13T19:25:13,071 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/ee2e09e61eac470b8ba4e0c48f7e20d5, entries=50, sequenceid=323, filesize=13.5 K 2024-11-13T19:25:13,073 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/B/ea88a4038f3a46ee8f0bf3a7b73b2214 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ea88a4038f3a46ee8f0bf3a7b73b2214 2024-11-13T19:25:13,078 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ea88a4038f3a46ee8f0bf3a7b73b2214, entries=50, sequenceid=323, filesize=7.2 K 2024-11-13T19:25:13,079 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/.tmp/C/77e6841cbef14d6f85687aad0305cece as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77e6841cbef14d6f85687aad0305cece 2024-11-13T19:25:13,083 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77e6841cbef14d6f85687aad0305cece, entries=50, sequenceid=323, filesize=7.2 K 2024-11-13T19:25:13,084 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for b04e7477f4d6e72d4397199d014ea80b in 1701ms, sequenceid=323, compaction requested=true 2024-11-13T19:25:13,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4656978ee89c49528a30a1a4f6608ec0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/44d68867118d41cf84f1554c246ba72d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4853e46af4b3400e924ad1cc8b4f3975, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/67fe23053d244263923655e747b4e2a9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/3ac935fcdf5847958d67466ff1a3ff4b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/a153f638afa8470e9a65bb32c90b1766, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0717adac96ce4b229c25ad339cf5426c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/820cc6b532fd461494654016310738cd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/99c983306bfe4ce5ae53caad299a3f06, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/27eba7ccc644493bbe400072c8e5b852, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0367befe73b34c4599c47328b0774e34, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/f294b92fc5ba478097821561f4a3b5b5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/bf5ef0d474774b5b966253f19c94dcbc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b5216390dbe34b61bbd0e15ba442896f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0caec87dff184530b6b4e4a1dcf2b2dc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/65661bbdef4d4fef89d6015d9ad2cfe5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/d296c09e660745198a079a1eaeeecd1a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b398d010ea1a4693bc7678e0fd5a65d2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/15c6caf29ec243a39b9e5993cac62f00] to archive 2024-11-13T19:25:13,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:25:13,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4656978ee89c49528a30a1a4f6608ec0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4656978ee89c49528a30a1a4f6608ec0 2024-11-13T19:25:13,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/44d68867118d41cf84f1554c246ba72d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/44d68867118d41cf84f1554c246ba72d 2024-11-13T19:25:13,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4853e46af4b3400e924ad1cc8b4f3975 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/4853e46af4b3400e924ad1cc8b4f3975 2024-11-13T19:25:13,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/67fe23053d244263923655e747b4e2a9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/67fe23053d244263923655e747b4e2a9 2024-11-13T19:25:13,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/3ac935fcdf5847958d67466ff1a3ff4b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/3ac935fcdf5847958d67466ff1a3ff4b 2024-11-13T19:25:13,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/a153f638afa8470e9a65bb32c90b1766 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/a153f638afa8470e9a65bb32c90b1766 2024-11-13T19:25:13,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0717adac96ce4b229c25ad339cf5426c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0717adac96ce4b229c25ad339cf5426c 2024-11-13T19:25:13,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/820cc6b532fd461494654016310738cd to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/820cc6b532fd461494654016310738cd 2024-11-13T19:25:13,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/99c983306bfe4ce5ae53caad299a3f06 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/99c983306bfe4ce5ae53caad299a3f06 2024-11-13T19:25:13,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/27eba7ccc644493bbe400072c8e5b852 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/27eba7ccc644493bbe400072c8e5b852 2024-11-13T19:25:13,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0367befe73b34c4599c47328b0774e34 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0367befe73b34c4599c47328b0774e34 2024-11-13T19:25:13,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/f294b92fc5ba478097821561f4a3b5b5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/f294b92fc5ba478097821561f4a3b5b5 2024-11-13T19:25:13,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/bf5ef0d474774b5b966253f19c94dcbc to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/bf5ef0d474774b5b966253f19c94dcbc 2024-11-13T19:25:13,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b5216390dbe34b61bbd0e15ba442896f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b5216390dbe34b61bbd0e15ba442896f 2024-11-13T19:25:13,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0caec87dff184530b6b4e4a1dcf2b2dc to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0caec87dff184530b6b4e4a1dcf2b2dc 2024-11-13T19:25:13,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/65661bbdef4d4fef89d6015d9ad2cfe5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/65661bbdef4d4fef89d6015d9ad2cfe5 2024-11-13T19:25:13,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/d296c09e660745198a079a1eaeeecd1a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/d296c09e660745198a079a1eaeeecd1a 2024-11-13T19:25:13,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b398d010ea1a4693bc7678e0fd5a65d2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/b398d010ea1a4693bc7678e0fd5a65d2 2024-11-13T19:25:13,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/15c6caf29ec243a39b9e5993cac62f00 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/15c6caf29ec243a39b9e5993cac62f00 2024-11-13T19:25:13,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/561403b86cea4a93927148ff5d9c0c72, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ab6ed6325d324f84ae2534fb8fc883a0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/8c40625900a9412aa3402753de8d2243, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/2bc3f47b56d44dc38d1cd9eb6e69282a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/894236f841b14534b8a3715f3fa58111, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a27c883267b945daa690a0b4ef163772, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/584babef65fc42dc9a6b83625bf2ff89, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/6e988d2562c64bc18cec9880f17e5537, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/d29916a7f06c4f0d861829e8ce017da3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ff013bc5d0a345d2a16f7547defcee6b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/cce8e028063f4025915b7d9e87bd0825, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/67710ec385a9404e8729eb3b5a0d5dd5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a8e376d231754a50af0580db77111f69, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/4703d779394846a5a1dfe7ee377098ec, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/91d9fd95f5be4cf185d94a8de72129fb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ccc6c4f3f89a495c99cd062be17b5de8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/dfab96578ad04e4098976ea4aa92c855, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/51e53b5ea6924cc0b938f4aae6a1ae24, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/823d7c654a7f4243ac63f1c6e80ae583] to archive 2024-11-13T19:25:13,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:25:13,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/561403b86cea4a93927148ff5d9c0c72 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/561403b86cea4a93927148ff5d9c0c72 2024-11-13T19:25:13,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ab6ed6325d324f84ae2534fb8fc883a0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ab6ed6325d324f84ae2534fb8fc883a0 2024-11-13T19:25:13,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/8c40625900a9412aa3402753de8d2243 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/8c40625900a9412aa3402753de8d2243 2024-11-13T19:25:13,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/2bc3f47b56d44dc38d1cd9eb6e69282a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/2bc3f47b56d44dc38d1cd9eb6e69282a 2024-11-13T19:25:13,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/894236f841b14534b8a3715f3fa58111 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/894236f841b14534b8a3715f3fa58111 2024-11-13T19:25:13,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a27c883267b945daa690a0b4ef163772 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a27c883267b945daa690a0b4ef163772 2024-11-13T19:25:13,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/584babef65fc42dc9a6b83625bf2ff89 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/584babef65fc42dc9a6b83625bf2ff89 2024-11-13T19:25:13,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/6e988d2562c64bc18cec9880f17e5537 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/6e988d2562c64bc18cec9880f17e5537 2024-11-13T19:25:13,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/d29916a7f06c4f0d861829e8ce017da3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/d29916a7f06c4f0d861829e8ce017da3 2024-11-13T19:25:13,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ff013bc5d0a345d2a16f7547defcee6b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ff013bc5d0a345d2a16f7547defcee6b 2024-11-13T19:25:13,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/cce8e028063f4025915b7d9e87bd0825 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/cce8e028063f4025915b7d9e87bd0825 2024-11-13T19:25:13,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/67710ec385a9404e8729eb3b5a0d5dd5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/67710ec385a9404e8729eb3b5a0d5dd5 2024-11-13T19:25:13,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a8e376d231754a50af0580db77111f69 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/a8e376d231754a50af0580db77111f69 2024-11-13T19:25:13,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/4703d779394846a5a1dfe7ee377098ec to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/4703d779394846a5a1dfe7ee377098ec 2024-11-13T19:25:13,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/91d9fd95f5be4cf185d94a8de72129fb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/91d9fd95f5be4cf185d94a8de72129fb 2024-11-13T19:25:13,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ccc6c4f3f89a495c99cd062be17b5de8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ccc6c4f3f89a495c99cd062be17b5de8 2024-11-13T19:25:13,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/dfab96578ad04e4098976ea4aa92c855 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/dfab96578ad04e4098976ea4aa92c855 2024-11-13T19:25:13,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/51e53b5ea6924cc0b938f4aae6a1ae24 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/51e53b5ea6924cc0b938f4aae6a1ae24 2024-11-13T19:25:13,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/823d7c654a7f4243ac63f1c6e80ae583 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/823d7c654a7f4243ac63f1c6e80ae583 2024-11-13T19:25:13,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3edbe1c57f6343b3b99bd6453bf8e51f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/206328ab78834bd2842052d44c3369ca, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/e2b75925c19f4e91ac8e7ac3c4dc066a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f3d6fea95c6b476a9c6ddf39a814cba6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6ee5ae9e4abd4f589caac1c83fc1e371, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/ca0d8f63759646d0a9773be8db805750, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/fe65953fc57148f09e65637c792d9a2f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/858d172942ce4666b5e766733e8571e1, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4e7b3b2904e04eadaf026faceb6df564, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f766ad121f7e4e68bae8193030efa1b7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77d1e94eadcd43fbac33f4af10957839, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0df4a2ae32d04bf8b358b62e02f1b7e6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3dbd3bebdbdd4d329652c5622816c327, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6477c6fa157a4db6aa37407bcf7f3561, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0cf563a2b0e14a3fa53c88d99202b9a2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4b66334ccd46467280a2819338e037d8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0f53c91755b949f1b9fc285de62a2178, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0ccc143b65bc42f6a7b3ef42a793cbc5] to archive 2024-11-13T19:25:13,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:25:13,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3edbe1c57f6343b3b99bd6453bf8e51f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3edbe1c57f6343b3b99bd6453bf8e51f 2024-11-13T19:25:13,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/206328ab78834bd2842052d44c3369ca to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/206328ab78834bd2842052d44c3369ca 2024-11-13T19:25:13,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/e2b75925c19f4e91ac8e7ac3c4dc066a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/e2b75925c19f4e91ac8e7ac3c4dc066a 2024-11-13T19:25:13,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f3d6fea95c6b476a9c6ddf39a814cba6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f3d6fea95c6b476a9c6ddf39a814cba6 2024-11-13T19:25:13,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6ee5ae9e4abd4f589caac1c83fc1e371 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6ee5ae9e4abd4f589caac1c83fc1e371 2024-11-13T19:25:13,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/ca0d8f63759646d0a9773be8db805750 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/ca0d8f63759646d0a9773be8db805750 2024-11-13T19:25:13,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/fe65953fc57148f09e65637c792d9a2f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/fe65953fc57148f09e65637c792d9a2f 2024-11-13T19:25:13,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/858d172942ce4666b5e766733e8571e1 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/858d172942ce4666b5e766733e8571e1 2024-11-13T19:25:13,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4e7b3b2904e04eadaf026faceb6df564 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4e7b3b2904e04eadaf026faceb6df564 2024-11-13T19:25:13,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f766ad121f7e4e68bae8193030efa1b7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/f766ad121f7e4e68bae8193030efa1b7 2024-11-13T19:25:13,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77d1e94eadcd43fbac33f4af10957839 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77d1e94eadcd43fbac33f4af10957839 2024-11-13T19:25:13,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0df4a2ae32d04bf8b358b62e02f1b7e6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0df4a2ae32d04bf8b358b62e02f1b7e6 2024-11-13T19:25:13,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3dbd3bebdbdd4d329652c5622816c327 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/3dbd3bebdbdd4d329652c5622816c327 2024-11-13T19:25:13,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6477c6fa157a4db6aa37407bcf7f3561 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/6477c6fa157a4db6aa37407bcf7f3561 2024-11-13T19:25:13,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0cf563a2b0e14a3fa53c88d99202b9a2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0cf563a2b0e14a3fa53c88d99202b9a2 2024-11-13T19:25:13,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4b66334ccd46467280a2819338e037d8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/4b66334ccd46467280a2819338e037d8 2024-11-13T19:25:13,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/5fd5cacf4dc74fdfa1cf7b1d6f412d5a 2024-11-13T19:25:13,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0f53c91755b949f1b9fc285de62a2178 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0f53c91755b949f1b9fc285de62a2178 2024-11-13T19:25:13,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0ccc143b65bc42f6a7b3ef42a793cbc5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/0ccc143b65bc42f6a7b3ef42a793cbc5 2024-11-13T19:25:13,148 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/recovered.edits/326.seqid, newMaxSeqId=326, maxSeqId=4 2024-11-13T19:25:13,148 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. 2024-11-13T19:25:13,148 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for b04e7477f4d6e72d4397199d014ea80b: Waiting for close lock at 1731525910926Running coprocessor pre-close hooks at 1731525910926Disabling compacts and flushes for region at 1731525910926Disabling writes for close at 1731525911383 (+457 ms)Obtaining lock to block concurrent updates at 1731525911383Preparing flush snapshotting stores in b04e7477f4d6e72d4397199d014ea80b at 1731525911383Finished memstore snapshotting TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1731525911383Flushing stores of TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b. at 1731525911384 (+1 ms)Flushing b04e7477f4d6e72d4397199d014ea80b/A: creating writer at 1731525911384Flushing mob file b04e7477f4d6e72d4397199d014ea80b/A: appending metadata at 1731525911390 (+6 ms)Flushing mob file b04e7477f4d6e72d4397199d014ea80b/A: closing flushed file at 1731525911390Flushing b04e7477f4d6e72d4397199d014ea80b/A: appending metadata at 1731525911809 (+419 ms)Flushing b04e7477f4d6e72d4397199d014ea80b/A: closing flushed file at 1731525911809Flushing b04e7477f4d6e72d4397199d014ea80b/B: creating writer at 1731525912227 (+418 ms)Flushing b04e7477f4d6e72d4397199d014ea80b/B: appending metadata at 1731525912229 (+2 ms)Flushing b04e7477f4d6e72d4397199d014ea80b/B: closing flushed file at 1731525912229Flushing b04e7477f4d6e72d4397199d014ea80b/C: creating writer at 1731525912644 (+415 ms)Flushing b04e7477f4d6e72d4397199d014ea80b/C: appending metadata at 1731525912647 (+3 ms)Flushing b04e7477f4d6e72d4397199d014ea80b/C: closing flushed file at 1731525912647Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bea069c: reopening flushed file at 1731525913063 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48b601b8: reopening flushed file at 1731525913071 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b8d9cc3: reopening flushed file at 1731525913078 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for b04e7477f4d6e72d4397199d014ea80b in 1701ms, sequenceid=323, compaction requested=true at 1731525913084 (+6 ms)Writing region close event to WAL at 1731525913145 (+61 ms)Running coprocessor post-close hooks at 1731525913148 (+3 ms)Closed at 1731525913148 2024-11-13T19:25:13,150 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:13,150 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=b04e7477f4d6e72d4397199d014ea80b, regionState=CLOSED 2024-11-13T19:25:13,151 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:25:13,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-13T19:25:13,153 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure b04e7477f4d6e72d4397199d014ea80b, server=e4b56bc8c4b3,40371,1731525792389 in 2.3780 sec 2024-11-13T19:25:13,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-13T19:25:13,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b04e7477f4d6e72d4397199d014ea80b, UNASSIGN in 2.3820 sec 2024-11-13T19:25:13,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-13T19:25:13,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3860 sec 2024-11-13T19:25:13,159 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525913159"}]},"ts":"1731525913159"} 2024-11-13T19:25:13,160 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:25:13,160 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:25:13,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3960 sec 2024-11-13T19:25:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-13T19:25:14,931 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:14,932 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:25:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:14,934 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T19:25:14,934 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:14,936 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,938 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/recovered.edits] 2024-11-13T19:25:14,941 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0c36c3f180bb42b09dde4e622aa21b0d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/0c36c3f180bb42b09dde4e622aa21b0d 2024-11-13T19:25:14,942 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/2221442175754ca99486545ec557f41e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/2221442175754ca99486545ec557f41e 2024-11-13T19:25:14,943 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/9a99873395d04510afca86cebf442443 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/9a99873395d04510afca86cebf442443 2024-11-13T19:25:14,944 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/ee2e09e61eac470b8ba4e0c48f7e20d5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/A/ee2e09e61eac470b8ba4e0c48f7e20d5 2024-11-13T19:25:14,946 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/1194ba540a2e45548b5977a5bb895041 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/1194ba540a2e45548b5977a5bb895041 2024-11-13T19:25:14,947 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0320143d5f846f0baaf4b4fbc548789 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0320143d5f846f0baaf4b4fbc548789 2024-11-13T19:25:14,948 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0cb77c16b4c4dafbb8bd4ff92a6a2bb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/e0cb77c16b4c4dafbb8bd4ff92a6a2bb 2024-11-13T19:25:14,948 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ea88a4038f3a46ee8f0bf3a7b73b2214 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/B/ea88a4038f3a46ee8f0bf3a7b73b2214 2024-11-13T19:25:14,950 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/1efc2d46927c4aa3b9053c69ad82c9b8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/1efc2d46927c4aa3b9053c69ad82c9b8 2024-11-13T19:25:14,951 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/512f7dc7cb224b519599f0d4deef8620 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/512f7dc7cb224b519599f0d4deef8620 2024-11-13T19:25:14,951 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77e6841cbef14d6f85687aad0305cece to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/77e6841cbef14d6f85687aad0305cece 2024-11-13T19:25:14,952 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/de309e47b3ec4918af5f9c12892c404a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/C/de309e47b3ec4918af5f9c12892c404a 2024-11-13T19:25:14,954 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/recovered.edits/326.seqid to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b/recovered.edits/326.seqid 2024-11-13T19:25:14,954 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,954 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:25:14,954 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:25:14,955 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T19:25:14,957 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131483aeb21d22409f835548bb34cf8170_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131483aeb21d22409f835548bb34cf8170_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,958 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111317a23186c6114935b0f4d773d04f52dd_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111317a23186c6114935b0f4d773d04f52dd_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,959 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111320e2ba76c8f5450ea5d6ffd7d297ffad_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111320e2ba76c8f5450ea5d6ffd7d297ffad_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,959 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111356d0e4b9eb604500a4f80857933ce08a_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111356d0e4b9eb604500a4f80857933ce08a_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,960 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136aa8b07a242046ebaf5c461ca81b58f1_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411136aa8b07a242046ebaf5c461ca81b58f1_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,961 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113741b24aa620a47cda4f284eb83c881cc_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113741b24aa620a47cda4f284eb83c881cc_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,962 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113898b8daddc614562abe1a3515c1964ce_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113898b8daddc614562abe1a3515c1964ce_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,963 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113927fd666564947ef8788cabceaa607cc_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113927fd666564947ef8788cabceaa607cc_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,963 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4e0087944f54040a16f28876155f70b_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a4e0087944f54040a16f28876155f70b_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,964 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a9fe6d782a174eb6ac949442cfab7d47_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113a9fe6d782a174eb6ac949442cfab7d47_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,965 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af48940845cb497897679f66c4ae6366_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af48940845cb497897679f66c4ae6366_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,965 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b0946ff5420f4ff4b743772e4d451bd9_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b0946ff5420f4ff4b743772e4d451bd9_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,966 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b9e5362c7b3e4960bbaa8391445ca472_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113b9e5362c7b3e4960bbaa8391445ca472_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,967 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c75e263d275c4816b82c10a99e66bb6b_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c75e263d275c4816b82c10a99e66bb6b_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,967 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc90c0dfbeb74d95a80a7cd55494b65a_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113dc90c0dfbeb74d95a80a7cd55494b65a_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,968 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e19627aba34e4abdae9cd03b81aa8258_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e19627aba34e4abdae9cd03b81aa8258_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,969 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e94fce7fc68f4fdf8ad615c085c44351_b04e7477f4d6e72d4397199d014ea80b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113e94fce7fc68f4fdf8ad615c085c44351_b04e7477f4d6e72d4397199d014ea80b 2024-11-13T19:25:14,969 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:25:14,970 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:14,972 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:25:14,973 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:25:14,974 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:14,974 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:25:14,974 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731525914974"}]},"ts":"9223372036854775807"} 2024-11-13T19:25:14,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:25:14,975 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b04e7477f4d6e72d4397199d014ea80b, NAME => 'TestAcidGuarantees,,1731525885576.b04e7477f4d6e72d4397199d014ea80b.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:25:14,975 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:25:14,976 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731525914975"}]},"ts":"9223372036854775807"} 2024-11-13T19:25:14,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:25:14,977 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:14,978 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-13T19:25:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-13T19:25:15,042 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:25:15,048 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:15,056 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 238) - Thread LEAK? -, OpenFileDescriptor=459 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=824 (was 1187), ProcessCount=11 (was 11), AvailableMemoryMB=2585 (was 2610) 2024-11-13T19:25:15,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=824, ProcessCount=11, AvailableMemoryMB=2584 2024-11-13T19:25:15,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:25:15,066 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:25:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:15,068 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:25:15,068 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:15,068 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-13T19:25:15,069 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:25:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T19:25:15,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742233_1409 (size=959) 2024-11-13T19:25:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T19:25:15,368 WARN [regionserver/e4b56bc8c4b3:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-13T19:25:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T19:25:15,479 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:25:15,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742234_1410 (size=53) 2024-11-13T19:25:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T19:25:15,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:25:15,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d17e3cc0ba121595297bf675d9f1d2e3, disabling compactions & flushes 2024-11-13T19:25:15,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:15,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:15,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. after waiting 0 ms 2024-11-13T19:25:15,890 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:15,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:15,891 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d17e3cc0ba121595297bf675d9f1d2e3: Waiting for close lock at 1731525915890Disabling compacts and flushes for region at 1731525915890Disabling writes for close at 1731525915890Writing region close event to WAL at 1731525915890Closed at 1731525915890 2024-11-13T19:25:15,937 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:25:15,938 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731525915937"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731525915937"}]},"ts":"1731525915937"} 2024-11-13T19:25:15,942 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:25:15,944 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:25:15,945 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525915944"}]},"ts":"1731525915944"} 2024-11-13T19:25:15,948 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:25:15,949 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, ASSIGN}] 2024-11-13T19:25:15,951 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, ASSIGN 2024-11-13T19:25:15,952 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, ASSIGN; state=OFFLINE, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=false 2024-11-13T19:25:16,103 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=d17e3cc0ba121595297bf675d9f1d2e3, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:16,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, ASSIGN because future has completed 2024-11-13T19:25:16,108 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:25:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T19:25:16,271 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:16,271 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:25:16,273 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,273 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:25:16,273 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,273 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,275 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,277 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d17e3cc0ba121595297bf675d9f1d2e3 columnFamilyName A 2024-11-13T19:25:16,277 DEBUG [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:16,278 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] regionserver.HStore(327): Store=d17e3cc0ba121595297bf675d9f1d2e3/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:16,278 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,279 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d17e3cc0ba121595297bf675d9f1d2e3 columnFamilyName B 2024-11-13T19:25:16,279 DEBUG [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:16,280 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] regionserver.HStore(327): Store=d17e3cc0ba121595297bf675d9f1d2e3/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:16,280 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,282 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d17e3cc0ba121595297bf675d9f1d2e3 columnFamilyName C 2024-11-13T19:25:16,282 DEBUG [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:16,282 INFO [StoreOpener-d17e3cc0ba121595297bf675d9f1d2e3-1 {}] regionserver.HStore(327): Store=d17e3cc0ba121595297bf675d9f1d2e3/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:16,282 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,283 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,284 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,285 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,285 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,286 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:25:16,288 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,291 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:25:16,292 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened d17e3cc0ba121595297bf675d9f1d2e3; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59548671, jitterRate=-0.1126556545495987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:25:16,292 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:16,293 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for d17e3cc0ba121595297bf675d9f1d2e3: Running coprocessor pre-open hook at 1731525916273Writing region info on filesystem at 1731525916273Initializing all the Stores at 1731525916275 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525916275Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525916275Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525916275Cleaning up temporary data from old regions at 1731525916285 (+10 ms)Running coprocessor post-open hooks at 1731525916292 (+7 ms)Region opened successfully at 1731525916293 (+1 ms) 2024-11-13T19:25:16,293 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., pid=118, masterSystemTime=1731525916261 2024-11-13T19:25:16,296 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:16,296 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:16,297 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=d17e3cc0ba121595297bf675d9f1d2e3, regionState=OPEN, openSeqNum=2, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:16,299 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:25:16,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-13T19:25:16,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 in 192 msec 2024-11-13T19:25:16,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-13T19:25:16,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, ASSIGN in 353 msec 2024-11-13T19:25:16,305 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:25:16,306 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525916305"}]},"ts":"1731525916305"} 2024-11-13T19:25:16,307 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:25:16,308 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:25:16,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2420 sec 2024-11-13T19:25:17,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-13T19:25:17,222 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:17,224 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6d55bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,224 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,225 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,226 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bce7ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,227 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,229 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46824, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,229 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1277c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,230 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,232 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,234 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@17be0796 2024-11-13T19:25:17,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:25:17,236 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:25:17,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0c05eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,238 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,238 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,239 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38f3ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,241 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30dfc8ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,242 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d7458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,242 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,242 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,243 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16d4e692, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,245 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb2054, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,246 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32c61cf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528537a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,248 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46894, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,249 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@393e2843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,249 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c296bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,250 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,250 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,250 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5629ac75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11885df7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47bc1e91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,253 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e324bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,254 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,254 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46940, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9f68d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672fe15e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,256 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6be67033, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,257 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@344b4ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71238b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,259 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd063a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,260 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,260 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,261 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@537211d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@625b319d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,261 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,261 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,262 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b9085c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,262 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,263 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@198a2f24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49cd7b43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,264 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,264 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,264 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,264 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228903e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5473d43e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,266 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73f5639f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,266 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:17,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:17,267 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b8c44d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:17,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:17,268 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:17,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c211a32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:17,269 DEBUG [Thread-1857 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,269 DEBUG [Thread-1872 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,270 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,270 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,270 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,271 DEBUG [Thread-1868 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,271 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46406, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,271 DEBUG [Thread-1874 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,272 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,273 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:25:17,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,275 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,275 DEBUG [Thread-1876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,275 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:17,276 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46448, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,276 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:17,276 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,276 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,276 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,280 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,283 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:17,284 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:17,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,287 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2] 2024-11-13T19:25:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:17,289 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:17,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:25:17,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/2dbda7a5f6034509a7cc274660e371bf is 50, key is test_row_0/A:col10/1731525917287/Put/seqid=0 2024-11-13T19:25:17,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742235_1411 (size=14341) 2024-11-13T19:25:17,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/2dbda7a5f6034509a7cc274660e371bf 2024-11-13T19:25:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525977321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525977326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525977327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525977332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525977333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/18b86c323e5442d0af0beee8108dc22d is 50, key is test_row_0/B:col10/1731525917287/Put/seqid=0 2024-11-13T19:25:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742236_1412 (size=12001) 2024-11-13T19:25:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:25:17,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:25:17,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:17,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525977441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525977442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525977442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525977442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525977451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:25:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:17,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:25:17,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525977652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525977652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525977652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525977652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525977661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,665 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:25:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,745 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/18b86c323e5442d0af0beee8108dc22d 2024-11-13T19:25:17,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/376afc0486dc4d66ab7e049969628ea5 is 50, key is test_row_0/C:col10/1731525917287/Put/seqid=0 2024-11-13T19:25:17,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742237_1413 (size=12001) 2024-11-13T19:25:17,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:25:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:17,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:17,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:25:17,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525977962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525977962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525977962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525977962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:17,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:17,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525977971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:17,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:25:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:18,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:18,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:18,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:18,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/376afc0486dc4d66ab7e049969628ea5 2024-11-13T19:25:18,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/2dbda7a5f6034509a7cc274660e371bf as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2dbda7a5f6034509a7cc274660e371bf 2024-11-13T19:25:18,174 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2dbda7a5f6034509a7cc274660e371bf, entries=200, sequenceid=12, filesize=14.0 K 2024-11-13T19:25:18,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/18b86c323e5442d0af0beee8108dc22d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/18b86c323e5442d0af0beee8108dc22d 2024-11-13T19:25:18,178 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/18b86c323e5442d0af0beee8108dc22d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:25:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/376afc0486dc4d66ab7e049969628ea5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/376afc0486dc4d66ab7e049969628ea5 2024-11-13T19:25:18,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/376afc0486dc4d66ab7e049969628ea5, entries=150, sequenceid=12, filesize=11.7 K 2024-11-13T19:25:18,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d17e3cc0ba121595297bf675d9f1d2e3 in 893ms, sequenceid=12, compaction requested=false 2024-11-13T19:25:18,182 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-13T19:25:18,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:18,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-13T19:25:18,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:18,197 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T19:25:18,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/9c697579b89c4f05bf13b2dce93597fa is 50, key is test_row_0/A:col10/1731525917331/Put/seqid=0 2024-11-13T19:25:18,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742238_1414 (size=12001) 2024-11-13T19:25:18,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:25:18,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:18,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:18,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525978485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525978488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,493 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525978489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525978490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525978491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525978601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525978601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525978601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525978601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525978602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,607 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,611 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/9c697579b89c4f05bf13b2dce93597fa 2024-11-13T19:25:18,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/656aaec0ef8b4d679d118b2d0a21d53a is 50, key is test_row_0/B:col10/1731525917331/Put/seqid=0 2024-11-13T19:25:18,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742239_1415 (size=12001) 2024-11-13T19:25:18,666 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:25:18,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525978814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525978815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525978815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525978815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525978815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:18,820 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,021 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/656aaec0ef8b4d679d118b2d0a21d53a 2024-11-13T19:25:19,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/1058a8ded8c54f07a186863c0370c474 is 50, key is test_row_0/C:col10/1731525917331/Put/seqid=0 2024-11-13T19:25:19,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742240_1416 (size=12001) 2024-11-13T19:25:19,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525979131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525979131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525979131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525979131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525979132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,138 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:25:19,431 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/1058a8ded8c54f07a186863c0370c474 2024-11-13T19:25:19,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/9c697579b89c4f05bf13b2dce93597fa as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/9c697579b89c4f05bf13b2dce93597fa 2024-11-13T19:25:19,438 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/9c697579b89c4f05bf13b2dce93597fa, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:25:19,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/656aaec0ef8b4d679d118b2d0a21d53a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/656aaec0ef8b4d679d118b2d0a21d53a 2024-11-13T19:25:19,443 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/656aaec0ef8b4d679d118b2d0a21d53a, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:25:19,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/1058a8ded8c54f07a186863c0370c474 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1058a8ded8c54f07a186863c0370c474 2024-11-13T19:25:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1058a8ded8c54f07a186863c0370c474, entries=150, sequenceid=37, filesize=11.7 K 2024-11-13T19:25:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for d17e3cc0ba121595297bf675d9f1d2e3 in 1252ms, sequenceid=37, compaction requested=false 2024-11-13T19:25:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-13T19:25:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-13T19:25:19,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-13T19:25:19,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-13T19:25:19,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1830 sec 2024-11-13T19:25:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:19,646 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:19,646 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T19:25:19,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/322905bb75d54290afd1c173e059eaf0 is 50, key is test_row_0/A:col10/1731525919646/Put/seqid=0 2024-11-13T19:25:19,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742241_1417 (size=14341) 2024-11-13T19:25:19,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525979683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525979684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525979712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525979712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,725 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525979713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525979822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525979822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525979831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525979831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:19,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525979831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:19,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525980032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525980032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525980041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525980041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525980041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,049 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,060 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/322905bb75d54290afd1c173e059eaf0 2024-11-13T19:25:20,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/dcb3f5306bd14e969b1ed35d25e95ca2 is 50, key is test_row_0/B:col10/1731525919646/Put/seqid=0 2024-11-13T19:25:20,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742242_1418 (size=12001) 2024-11-13T19:25:20,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525980342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525980342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525980351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525980351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525980361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/dcb3f5306bd14e969b1ed35d25e95ca2 2024-11-13T19:25:20,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/b6e9b3e8cf0c4260a77c564556003f66 is 50, key is test_row_0/C:col10/1731525919646/Put/seqid=0 2024-11-13T19:25:20,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742243_1419 (size=12001) 2024-11-13T19:25:20,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525980851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525980851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525980861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525980861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:20,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525980872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:20,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:20,883 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/b6e9b3e8cf0c4260a77c564556003f66 2024-11-13T19:25:20,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/322905bb75d54290afd1c173e059eaf0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/322905bb75d54290afd1c173e059eaf0 2024-11-13T19:25:20,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/322905bb75d54290afd1c173e059eaf0, entries=200, sequenceid=51, filesize=14.0 K 2024-11-13T19:25:20,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/dcb3f5306bd14e969b1ed35d25e95ca2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/dcb3f5306bd14e969b1ed35d25e95ca2 2024-11-13T19:25:20,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/dcb3f5306bd14e969b1ed35d25e95ca2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T19:25:20,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/b6e9b3e8cf0c4260a77c564556003f66 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b6e9b3e8cf0c4260a77c564556003f66 2024-11-13T19:25:20,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b6e9b3e8cf0c4260a77c564556003f66, entries=150, sequenceid=51, filesize=11.7 K 2024-11-13T19:25:20,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for d17e3cc0ba121595297bf675d9f1d2e3 in 1289ms, sequenceid=51, compaction requested=true 2024-11-13T19:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:20,935 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:20,935 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:20,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:20,936 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:20,936 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:20,936 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/B is initiating minor compaction (all files) 2024-11-13T19:25:20,936 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/A is initiating minor compaction (all files) 2024-11-13T19:25:20,936 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/B in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:20,936 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/A in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:20,936 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/18b86c323e5442d0af0beee8108dc22d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/656aaec0ef8b4d679d118b2d0a21d53a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/dcb3f5306bd14e969b1ed35d25e95ca2] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.2 K 2024-11-13T19:25:20,936 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2dbda7a5f6034509a7cc274660e371bf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/9c697579b89c4f05bf13b2dce93597fa, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/322905bb75d54290afd1c173e059eaf0] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=39.7 K 2024-11-13T19:25:20,936 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b86c323e5442d0af0beee8108dc22d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731525917277 2024-11-13T19:25:20,936 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dbda7a5f6034509a7cc274660e371bf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731525917277 2024-11-13T19:25:20,937 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 656aaec0ef8b4d679d118b2d0a21d53a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525917317 2024-11-13T19:25:20,937 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c697579b89c4f05bf13b2dce93597fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525917317 2024-11-13T19:25:20,937 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb3f5306bd14e969b1ed35d25e95ca2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525918488 2024-11-13T19:25:20,937 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 322905bb75d54290afd1c173e059eaf0, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525918486 2024-11-13T19:25:20,942 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#B#compaction#351 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:20,942 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/7fcf72c77d9e469487cd645a14d537af is 50, key is test_row_0/B:col10/1731525919646/Put/seqid=0 2024-11-13T19:25:20,943 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#A#compaction#352 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:20,943 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/95eca11188dd42ab89e5246f0aac513c is 50, key is test_row_0/A:col10/1731525919646/Put/seqid=0 2024-11-13T19:25:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742244_1420 (size=12104) 2024-11-13T19:25:20,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742245_1421 (size=12104) 2024-11-13T19:25:20,949 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/7fcf72c77d9e469487cd645a14d537af as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7fcf72c77d9e469487cd645a14d537af 2024-11-13T19:25:20,954 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/B of d17e3cc0ba121595297bf675d9f1d2e3 into 7fcf72c77d9e469487cd645a14d537af(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:20,954 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:20,954 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/B, priority=13, startTime=1731525920935; duration=0sec 2024-11-13T19:25:20,954 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:20,954 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:B 2024-11-13T19:25:20,954 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:20,955 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:20,955 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/C is initiating minor compaction (all files) 2024-11-13T19:25:20,955 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/C in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:20,955 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/376afc0486dc4d66ab7e049969628ea5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1058a8ded8c54f07a186863c0370c474, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b6e9b3e8cf0c4260a77c564556003f66] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.2 K 2024-11-13T19:25:20,955 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 376afc0486dc4d66ab7e049969628ea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731525917277 2024-11-13T19:25:20,956 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1058a8ded8c54f07a186863c0370c474, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731525917317 2024-11-13T19:25:20,956 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting b6e9b3e8cf0c4260a77c564556003f66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525918488 2024-11-13T19:25:20,961 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#C#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:20,962 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/8a76c6737c3b40249394803a6e316300 is 50, key is test_row_0/C:col10/1731525919646/Put/seqid=0 2024-11-13T19:25:20,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742246_1422 (size=12104) 2024-11-13T19:25:21,350 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/95eca11188dd42ab89e5246f0aac513c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/95eca11188dd42ab89e5246f0aac513c 2024-11-13T19:25:21,354 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/A of d17e3cc0ba121595297bf675d9f1d2e3 into 95eca11188dd42ab89e5246f0aac513c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:21,354 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:21,354 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/A, priority=13, startTime=1731525920935; duration=0sec 2024-11-13T19:25:21,354 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:21,354 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:A 2024-11-13T19:25:21,372 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/8a76c6737c3b40249394803a6e316300 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8a76c6737c3b40249394803a6e316300 2024-11-13T19:25:21,376 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/C of d17e3cc0ba121595297bf675d9f1d2e3 into 8a76c6737c3b40249394803a6e316300(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:21,376 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:21,376 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/C, priority=13, startTime=1731525920935; duration=0sec 2024-11-13T19:25:21,376 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:21,376 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:C 2024-11-13T19:25:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-13T19:25:21,431 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:21,431 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-13T19:25:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:25:21,433 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:21,434 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:21,434 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:25:21,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-13T19:25:21,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:21,586 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T19:25:21,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/236cc9a5db4841cb8adcfe9900f4e6a2 is 50, key is test_row_0/A:col10/1731525919692/Put/seqid=0 2024-11-13T19:25:21,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742247_1423 (size=12001) 2024-11-13T19:25:21,600 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/236cc9a5db4841cb8adcfe9900f4e6a2 2024-11-13T19:25:21,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/cc0a931ac70642b991ddf6dc3746e637 is 50, key is test_row_0/B:col10/1731525919692/Put/seqid=0 2024-11-13T19:25:21,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742248_1424 (size=12001) 2024-11-13T19:25:21,613 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/cc0a931ac70642b991ddf6dc3746e637 2024-11-13T19:25:21,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/0302e4d5c8074032ab94d92ffd64b532 is 50, key is test_row_0/C:col10/1731525919692/Put/seqid=0 2024-11-13T19:25:21,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742249_1425 (size=12001) 2024-11-13T19:25:21,628 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/0302e4d5c8074032ab94d92ffd64b532 2024-11-13T19:25:21,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/236cc9a5db4841cb8adcfe9900f4e6a2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/236cc9a5db4841cb8adcfe9900f4e6a2 2024-11-13T19:25:21,634 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/236cc9a5db4841cb8adcfe9900f4e6a2, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T19:25:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/cc0a931ac70642b991ddf6dc3746e637 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/cc0a931ac70642b991ddf6dc3746e637 2024-11-13T19:25:21,638 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/cc0a931ac70642b991ddf6dc3746e637, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T19:25:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/0302e4d5c8074032ab94d92ffd64b532 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/0302e4d5c8074032ab94d92ffd64b532 2024-11-13T19:25:21,642 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/0302e4d5c8074032ab94d92ffd64b532, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T19:25:21,643 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for d17e3cc0ba121595297bf675d9f1d2e3 in 57ms, sequenceid=77, compaction requested=false 2024-11-13T19:25:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:21,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-13T19:25:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-13T19:25:21,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-13T19:25:21,646 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 210 msec 2024-11-13T19:25:21,649 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 215 msec 2024-11-13T19:25:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-13T19:25:21,751 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:21,752 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-13T19:25:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:25:21,754 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:21,755 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:21,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:25:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:21,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:21,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:25:21,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/7b6984d22b12405fa7712649998cb1b7 is 50, key is test_row_0/A:col10/1731525921868/Put/seqid=0 2024-11-13T19:25:21,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742250_1426 (size=16681) 2024-11-13T19:25:21,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:21,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:21,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:21,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525981909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:21,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525981916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:21,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:21,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525981918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:21,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525981918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,929 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525981919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525982031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525982031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525982031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525982032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525982041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:25:22,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525982242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525982242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525982242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,248 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525982251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525982251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/7b6984d22b12405fa7712649998cb1b7 2024-11-13T19:25:22,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/a34735de407743c3bb466ee2fcf3c332 is 50, key is test_row_0/B:col10/1731525921868/Put/seqid=0 2024-11-13T19:25:22,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742251_1427 (size=12001) 2024-11-13T19:25:22,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:22,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:25:22,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525982551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525982552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525982561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525982561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525982561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:22,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:22,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/a34735de407743c3bb466ee2fcf3c332 2024-11-13T19:25:22,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/9a0f86e397514ba6a7e22c7075f22e74 is 50, key is test_row_0/C:col10/1731525921868/Put/seqid=0 2024-11-13T19:25:22,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742252_1428 (size=12001) 2024-11-13T19:25:22,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:25:22,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:22,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:23,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525983062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:23,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525983071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525983071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525983071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,077 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:23,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525983071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:23,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:23,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/9a0f86e397514ba6a7e22c7075f22e74 2024-11-13T19:25:23,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/7b6984d22b12405fa7712649998cb1b7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7b6984d22b12405fa7712649998cb1b7 2024-11-13T19:25:23,110 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7b6984d22b12405fa7712649998cb1b7, entries=250, sequenceid=88, filesize=16.3 K 2024-11-13T19:25:23,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/a34735de407743c3bb466ee2fcf3c332 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a34735de407743c3bb466ee2fcf3c332 2024-11-13T19:25:23,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a34735de407743c3bb466ee2fcf3c332, entries=150, sequenceid=88, filesize=11.7 K 2024-11-13T19:25:23,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/9a0f86e397514ba6a7e22c7075f22e74 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9a0f86e397514ba6a7e22c7075f22e74 2024-11-13T19:25:23,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9a0f86e397514ba6a7e22c7075f22e74, entries=150, sequenceid=88, filesize=11.7 K 2024-11-13T19:25:23,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d17e3cc0ba121595297bf675d9f1d2e3 in 1245ms, sequenceid=88, compaction requested=true 2024-11-13T19:25:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:23,118 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:23,118 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:23,118 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/B is initiating minor compaction (all files) 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/A is initiating minor compaction (all files) 2024-11-13T19:25:23,119 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/B in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:23,119 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/A in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:23,119 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/95eca11188dd42ab89e5246f0aac513c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/236cc9a5db4841cb8adcfe9900f4e6a2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7b6984d22b12405fa7712649998cb1b7] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=39.8 K 2024-11-13T19:25:23,119 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7fcf72c77d9e469487cd645a14d537af, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/cc0a931ac70642b991ddf6dc3746e637, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a34735de407743c3bb466ee2fcf3c332] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.3 K 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fcf72c77d9e469487cd645a14d537af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525918488 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95eca11188dd42ab89e5246f0aac513c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525918488 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 236cc9a5db4841cb8adcfe9900f4e6a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525919681 2024-11-13T19:25:23,119 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting cc0a931ac70642b991ddf6dc3746e637, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525919681 2024-11-13T19:25:23,120 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b6984d22b12405fa7712649998cb1b7, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731525921863 2024-11-13T19:25:23,120 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a34735de407743c3bb466ee2fcf3c332, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731525921868 2024-11-13T19:25:23,124 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:23,124 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/ce88e46ad93e49769b64fb6e2aa99645 is 50, key is test_row_0/B:col10/1731525921868/Put/seqid=0 2024-11-13T19:25:23,128 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#A#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:23,128 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/e46a184f569546b4992b42dd97eb6404 is 50, key is test_row_0/A:col10/1731525921868/Put/seqid=0 2024-11-13T19:25:23,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-13T19:25:23,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:23,130 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-13T19:25:23,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742253_1429 (size=12207) 2024-11-13T19:25:23,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742254_1430 (size=12207) 2024-11-13T19:25:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a is 50, key is test_row_0/A:col10/1731525921916/Put/seqid=0 2024-11-13T19:25:23,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742255_1431 (size=12001) 2024-11-13T19:25:23,146 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a 2024-11-13T19:25:23,147 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/e46a184f569546b4992b42dd97eb6404 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/e46a184f569546b4992b42dd97eb6404 2024-11-13T19:25:23,150 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/A of d17e3cc0ba121595297bf675d9f1d2e3 into e46a184f569546b4992b42dd97eb6404(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:23,151 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:23,151 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/A, priority=13, startTime=1731525923118; duration=0sec 2024-11-13T19:25:23,151 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:23,151 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:A 2024-11-13T19:25:23,151 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:23,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/65b9ee4dc1c04b47a626449e7afc338f is 50, key is test_row_0/B:col10/1731525921916/Put/seqid=0 2024-11-13T19:25:23,154 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:23,154 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/C is initiating minor compaction (all files) 2024-11-13T19:25:23,155 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/C in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:23,155 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8a76c6737c3b40249394803a6e316300, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/0302e4d5c8074032ab94d92ffd64b532, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9a0f86e397514ba6a7e22c7075f22e74] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.3 K 2024-11-13T19:25:23,155 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a76c6737c3b40249394803a6e316300, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731525918488 2024-11-13T19:25:23,155 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0302e4d5c8074032ab94d92ffd64b532, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525919681 2024-11-13T19:25:23,155 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a0f86e397514ba6a7e22c7075f22e74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731525921868 2024-11-13T19:25:23,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742256_1432 (size=12001) 2024-11-13T19:25:23,161 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#C#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:23,161 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/7fc44b05268f478fb75f32a67bafff6d is 50, key is test_row_0/C:col10/1731525921868/Put/seqid=0 2024-11-13T19:25:23,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742257_1433 (size=12207) 2024-11-13T19:25:23,535 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/ce88e46ad93e49769b64fb6e2aa99645 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ce88e46ad93e49769b64fb6e2aa99645 2024-11-13T19:25:23,539 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/B of d17e3cc0ba121595297bf675d9f1d2e3 into ce88e46ad93e49769b64fb6e2aa99645(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:23,539 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:23,539 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/B, priority=13, startTime=1731525923118; duration=0sec 2024-11-13T19:25:23,539 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:23,539 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:B 2024-11-13T19:25:23,557 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/65b9ee4dc1c04b47a626449e7afc338f 2024-11-13T19:25:23,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/cd32fc8c952f4eb9b62be4f180c055c2 is 50, key is test_row_0/C:col10/1731525921916/Put/seqid=0 2024-11-13T19:25:23,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742258_1434 (size=12001) 2024-11-13T19:25:23,575 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/7fc44b05268f478fb75f32a67bafff6d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/7fc44b05268f478fb75f32a67bafff6d 2024-11-13T19:25:23,579 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/C of d17e3cc0ba121595297bf675d9f1d2e3 into 7fc44b05268f478fb75f32a67bafff6d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:23,579 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:23,579 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/C, priority=13, startTime=1731525923118; duration=0sec 2024-11-13T19:25:23,579 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:23,579 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:C 2024-11-13T19:25:23,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:25:23,969 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/cd32fc8c952f4eb9b62be4f180c055c2 2024-11-13T19:25:23,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a 2024-11-13T19:25:23,976 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T19:25:23,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/65b9ee4dc1c04b47a626449e7afc338f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/65b9ee4dc1c04b47a626449e7afc338f 2024-11-13T19:25:23,980 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/65b9ee4dc1c04b47a626449e7afc338f, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T19:25:23,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/cd32fc8c952f4eb9b62be4f180c055c2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/cd32fc8c952f4eb9b62be4f180c055c2 2024-11-13T19:25:23,984 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/cd32fc8c952f4eb9b62be4f180c055c2, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T19:25:23,984 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for d17e3cc0ba121595297bf675d9f1d2e3 in 854ms, sequenceid=113, compaction requested=false 2024-11-13T19:25:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:23,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-13T19:25:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-13T19:25:23,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-13T19:25:23,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2310 sec 2024-11-13T19:25:23,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.2360 sec 2024-11-13T19:25:24,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:24,093 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:24,093 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-13T19:25:24,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/0c291b2853154cb8a609651991643258 is 50, key is test_row_0/A:col10/1731525924089/Put/seqid=0 2024-11-13T19:25:24,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742259_1435 (size=19021) 2024-11-13T19:25:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525984121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525984122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525984125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,131 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525984126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525984126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525984231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525984232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525984242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525984242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525984242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525984442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525984442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525984451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525984451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525984461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/0c291b2853154cb8a609651991643258 2024-11-13T19:25:24,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/1fff185962d6466cbd13022de644e41d is 50, key is test_row_0/B:col10/1731525924089/Put/seqid=0 2024-11-13T19:25:24,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742260_1436 (size=12001) 2024-11-13T19:25:24,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525984751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525984762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525984762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525984771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525984771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:24,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:24,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/1fff185962d6466cbd13022de644e41d 2024-11-13T19:25:24,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/f0f805e29b614364b93923b8170512d3 is 50, key is test_row_0/C:col10/1731525924089/Put/seqid=0 2024-11-13T19:25:24,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742261_1437 (size=12001) 2024-11-13T19:25:25,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:25,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525985262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525985281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:25,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525985282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525985282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:25,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:25,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525985292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:25,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:25,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:25,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/f0f805e29b614364b93923b8170512d3 2024-11-13T19:25:25,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/0c291b2853154cb8a609651991643258 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/0c291b2853154cb8a609651991643258 2024-11-13T19:25:25,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/0c291b2853154cb8a609651991643258, entries=300, sequenceid=128, filesize=18.6 K 2024-11-13T19:25:25,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/1fff185962d6466cbd13022de644e41d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/1fff185962d6466cbd13022de644e41d 2024-11-13T19:25:25,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/1fff185962d6466cbd13022de644e41d, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T19:25:25,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/f0f805e29b614364b93923b8170512d3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f0f805e29b614364b93923b8170512d3 2024-11-13T19:25:25,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f0f805e29b614364b93923b8170512d3, entries=150, sequenceid=128, filesize=11.7 K 2024-11-13T19:25:25,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d17e3cc0ba121595297bf675d9f1d2e3 in 1247ms, sequenceid=128, compaction requested=true 2024-11-13T19:25:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:25,340 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:25,340 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:25,341 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:25,341 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:25,341 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/A is initiating minor compaction (all files) 2024-11-13T19:25:25,341 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/B is initiating minor compaction (all files) 2024-11-13T19:25:25,341 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/A in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:25,341 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/B in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:25,341 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ce88e46ad93e49769b64fb6e2aa99645, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/65b9ee4dc1c04b47a626449e7afc338f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/1fff185962d6466cbd13022de644e41d] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.4 K 2024-11-13T19:25:25,341 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/e46a184f569546b4992b42dd97eb6404, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/0c291b2853154cb8a609651991643258] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=42.2 K 2024-11-13T19:25:25,341 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting e46a184f569546b4992b42dd97eb6404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731525921868 2024-11-13T19:25:25,341 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting ce88e46ad93e49769b64fb6e2aa99645, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731525921868 2024-11-13T19:25:25,342 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 65b9ee4dc1c04b47a626449e7afc338f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731525921916 2024-11-13T19:25:25,342 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe4e35e1bf4c49c2ad7c4bb38c6e5a4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731525921916 2024-11-13T19:25:25,342 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fff185962d6466cbd13022de644e41d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731525924085 2024-11-13T19:25:25,342 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c291b2853154cb8a609651991643258, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731525924085 2024-11-13T19:25:25,349 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#A#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:25,350 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/384e05afd5d74a22933c6baed0f552ea is 50, key is test_row_0/A:col10/1731525924089/Put/seqid=0 2024-11-13T19:25:25,357 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#B#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:25,357 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/6d607dbffeae4df999d6c4752a4cb5d4 is 50, key is test_row_0/B:col10/1731525924089/Put/seqid=0 2024-11-13T19:25:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742262_1438 (size=12309) 2024-11-13T19:25:25,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742263_1439 (size=12309) 2024-11-13T19:25:25,373 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/6d607dbffeae4df999d6c4752a4cb5d4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/6d607dbffeae4df999d6c4752a4cb5d4 2024-11-13T19:25:25,378 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/B of d17e3cc0ba121595297bf675d9f1d2e3 into 6d607dbffeae4df999d6c4752a4cb5d4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:25,378 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:25,378 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/B, priority=13, startTime=1731525925340; duration=0sec 2024-11-13T19:25:25,378 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:25,378 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:B 2024-11-13T19:25:25,378 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:25,378 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:25,379 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/C is initiating minor compaction (all files) 2024-11-13T19:25:25,379 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/C in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:25,379 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/7fc44b05268f478fb75f32a67bafff6d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/cd32fc8c952f4eb9b62be4f180c055c2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f0f805e29b614364b93923b8170512d3] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.4 K 2024-11-13T19:25:25,379 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc44b05268f478fb75f32a67bafff6d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731525921868 2024-11-13T19:25:25,380 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting cd32fc8c952f4eb9b62be4f180c055c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731525921916 2024-11-13T19:25:25,382 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting f0f805e29b614364b93923b8170512d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731525924085 2024-11-13T19:25:25,387 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#C#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:25,388 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/a08f1b111f174deb892b27e579c8857e is 50, key is test_row_0/C:col10/1731525924089/Put/seqid=0 2024-11-13T19:25:25,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742264_1440 (size=12309) 2024-11-13T19:25:25,772 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/384e05afd5d74a22933c6baed0f552ea as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/384e05afd5d74a22933c6baed0f552ea 2024-11-13T19:25:25,775 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/A of d17e3cc0ba121595297bf675d9f1d2e3 into 384e05afd5d74a22933c6baed0f552ea(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:25,775 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:25,775 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/A, priority=13, startTime=1731525925340; duration=0sec 2024-11-13T19:25:25,775 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:25,775 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:A 2024-11-13T19:25:25,802 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/a08f1b111f174deb892b27e579c8857e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a08f1b111f174deb892b27e579c8857e 2024-11-13T19:25:25,806 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/C of d17e3cc0ba121595297bf675d9f1d2e3 into a08f1b111f174deb892b27e579c8857e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:25,806 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:25,806 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/C, priority=13, startTime=1731525925340; duration=0sec 2024-11-13T19:25:25,808 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:25,808 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:C 2024-11-13T19:25:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-13T19:25:25,912 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:25,912 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-13T19:25:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:25:25,914 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:25,914 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:25,914 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:25:26,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-13T19:25:26,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:26,066 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-13T19:25:26,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/d1e23127514f42e59c50adb9c5bc06b6 is 50, key is test_row_0/A:col10/1731525924115/Put/seqid=0 2024-11-13T19:25:26,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742265_1441 (size=12151) 2024-11-13T19:25:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:25:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:26,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:26,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525986305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525986321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525986334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525986334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,346 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525986334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525986441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525986452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525986452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525986452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,476 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/d1e23127514f42e59c50adb9c5bc06b6 2024-11-13T19:25:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/a1764e86b7864a2b98509821609ba3ad is 50, key is test_row_0/B:col10/1731525924115/Put/seqid=0 2024-11-13T19:25:26,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742266_1442 (size=12151) 2024-11-13T19:25:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:25:26,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525986651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525986672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525986672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525986672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,894 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/a1764e86b7864a2b98509821609ba3ad 2024-11-13T19:25:26,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/49fc633589c54b21b5a2b84fc82af3c1 is 50, key is test_row_0/C:col10/1731525924115/Put/seqid=0 2024-11-13T19:25:26,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742267_1443 (size=12151) 2024-11-13T19:25:26,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525986961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525986991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525986991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525986991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:25:27,309 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/49fc633589c54b21b5a2b84fc82af3c1 2024-11-13T19:25:27,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/d1e23127514f42e59c50adb9c5bc06b6 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d1e23127514f42e59c50adb9c5bc06b6 2024-11-13T19:25:27,316 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d1e23127514f42e59c50adb9c5bc06b6, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T19:25:27,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/a1764e86b7864a2b98509821609ba3ad as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a1764e86b7864a2b98509821609ba3ad 2024-11-13T19:25:27,320 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a1764e86b7864a2b98509821609ba3ad, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T19:25:27,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/49fc633589c54b21b5a2b84fc82af3c1 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/49fc633589c54b21b5a2b84fc82af3c1 2024-11-13T19:25:27,324 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/49fc633589c54b21b5a2b84fc82af3c1, entries=150, sequenceid=155, filesize=11.9 K 2024-11-13T19:25:27,325 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d17e3cc0ba121595297bf675d9f1d2e3 in 1258ms, sequenceid=155, compaction requested=false 2024-11-13T19:25:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:27,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-13T19:25:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-13T19:25:27,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-13T19:25:27,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-13T19:25:27,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4160 sec 2024-11-13T19:25:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:27,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:27,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-13T19:25:27,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/bd24e589f82440cfa1896b686e9f4f0c is 50, key is test_row_0/A:col10/1731525926317/Put/seqid=0 2024-11-13T19:25:27,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742268_1444 (size=14541) 2024-11-13T19:25:27,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525987534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525987539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,547 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525987539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525987544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525987652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525987652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525987652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525987652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525987862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525987862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525987871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525987871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:27,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:27,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/bd24e589f82440cfa1896b686e9f4f0c 2024-11-13T19:25:27,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/770e0687cddf4c7891283f7f8add3feb is 50, key is test_row_0/B:col10/1731525926317/Put/seqid=0 2024-11-13T19:25:27,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742269_1445 (size=12151) 2024-11-13T19:25:28,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-13T19:25:28,062 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:28,062 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-13T19:25:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:25:28,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:28,064 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:28,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:28,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:25:28,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525988171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525988171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,179 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525988182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525988182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:25:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/770e0687cddf4c7891283f7f8add3feb 2024-11-13T19:25:28,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/8f8c908598d943cfb153d57684276272 is 50, key is test_row_0/C:col10/1731525926317/Put/seqid=0 2024-11-13T19:25:28,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742270_1446 (size=12151) 2024-11-13T19:25:28,369 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:25:28,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:28,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525988361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,371 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:25:28,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:25:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:25:28,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:28,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:25:28,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525988682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525988691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525988691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:28,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525988701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:28,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:28,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/8f8c908598d943cfb153d57684276272 2024-11-13T19:25:28,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/bd24e589f82440cfa1896b686e9f4f0c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/bd24e589f82440cfa1896b686e9f4f0c 2024-11-13T19:25:28,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/bd24e589f82440cfa1896b686e9f4f0c, entries=200, sequenceid=168, filesize=14.2 K 2024-11-13T19:25:28,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/770e0687cddf4c7891283f7f8add3feb as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/770e0687cddf4c7891283f7f8add3feb 2024-11-13T19:25:28,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/770e0687cddf4c7891283f7f8add3feb, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T19:25:28,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/8f8c908598d943cfb153d57684276272 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8f8c908598d943cfb153d57684276272 2024-11-13T19:25:28,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8f8c908598d943cfb153d57684276272, entries=150, sequenceid=168, filesize=11.9 K 2024-11-13T19:25:28,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d17e3cc0ba121595297bf675d9f1d2e3 in 1250ms, sequenceid=168, compaction requested=true 2024-11-13T19:25:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:28,736 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:28,736 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:28,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/A is initiating minor compaction (all files) 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/B is initiating minor compaction (all files) 2024-11-13T19:25:28,737 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/A in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,737 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/B in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,737 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/384e05afd5d74a22933c6baed0f552ea, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d1e23127514f42e59c50adb9c5bc06b6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/bd24e589f82440cfa1896b686e9f4f0c] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=38.1 K 2024-11-13T19:25:28,737 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/6d607dbffeae4df999d6c4752a4cb5d4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a1764e86b7864a2b98509821609ba3ad, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/770e0687cddf4c7891283f7f8add3feb] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.8 K 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384e05afd5d74a22933c6baed0f552ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731525924085 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d607dbffeae4df999d6c4752a4cb5d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731525924085 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1e23127514f42e59c50adb9c5bc06b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731525924115 2024-11-13T19:25:28,737 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting a1764e86b7864a2b98509821609ba3ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731525924115 2024-11-13T19:25:28,738 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd24e589f82440cfa1896b686e9f4f0c, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731525926304 2024-11-13T19:25:28,738 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 770e0687cddf4c7891283f7f8add3feb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731525926304 2024-11-13T19:25:28,742 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#B#compaction#378 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:28,743 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#A#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:28,743 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/5465fa9111a244f682af34075f557dd7 is 50, key is test_row_0/B:col10/1731525926317/Put/seqid=0 2024-11-13T19:25:28,743 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/cc9f5838accb4e0591b396d580e3c075 is 50, key is test_row_0/A:col10/1731525926317/Put/seqid=0 2024-11-13T19:25:28,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742271_1447 (size=12561) 2024-11-13T19:25:28,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742272_1448 (size=12561) 2024-11-13T19:25:28,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-13T19:25:28,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:28,827 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T19:25:28,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/abe98e1c48cd45c7934cf6b3d3dd7357 is 50, key is test_row_0/A:col10/1731525927537/Put/seqid=0 2024-11-13T19:25:28,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742273_1449 (size=12151) 2024-11-13T19:25:29,150 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/cc9f5838accb4e0591b396d580e3c075 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/cc9f5838accb4e0591b396d580e3c075 2024-11-13T19:25:29,153 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/A of d17e3cc0ba121595297bf675d9f1d2e3 into cc9f5838accb4e0591b396d580e3c075(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:29,153 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:29,153 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/A, priority=13, startTime=1731525928736; duration=0sec 2024-11-13T19:25:29,153 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:29,153 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:A 2024-11-13T19:25:29,154 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:29,154 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:29,154 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/C is initiating minor compaction (all files) 2024-11-13T19:25:29,154 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/C in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:29,154 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a08f1b111f174deb892b27e579c8857e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/49fc633589c54b21b5a2b84fc82af3c1, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8f8c908598d943cfb153d57684276272] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=35.8 K 2024-11-13T19:25:29,155 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08f1b111f174deb892b27e579c8857e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731525924085 2024-11-13T19:25:29,155 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49fc633589c54b21b5a2b84fc82af3c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731525924115 2024-11-13T19:25:29,155 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f8c908598d943cfb153d57684276272, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731525926304 2024-11-13T19:25:29,156 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/5465fa9111a244f682af34075f557dd7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/5465fa9111a244f682af34075f557dd7 2024-11-13T19:25:29,160 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/B of d17e3cc0ba121595297bf675d9f1d2e3 into 5465fa9111a244f682af34075f557dd7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:29,160 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:29,160 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/B, priority=13, startTime=1731525928736; duration=0sec 2024-11-13T19:25:29,160 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:29,160 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:B 2024-11-13T19:25:29,170 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#C#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:29,170 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/a5fa7a79d96d4462ad6509b2b05a4c85 is 50, key is test_row_0/C:col10/1731525926317/Put/seqid=0 2024-11-13T19:25:29,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742274_1450 (size=12561) 2024-11-13T19:25:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:25:29,234 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/abe98e1c48cd45c7934cf6b3d3dd7357 2024-11-13T19:25:29,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/8348ef5e68474188b333eb3b6117757a is 50, key is test_row_0/B:col10/1731525927537/Put/seqid=0 2024-11-13T19:25:29,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742275_1451 (size=12151) 2024-11-13T19:25:29,582 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/a5fa7a79d96d4462ad6509b2b05a4c85 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a5fa7a79d96d4462ad6509b2b05a4c85 2024-11-13T19:25:29,585 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/C of d17e3cc0ba121595297bf675d9f1d2e3 into a5fa7a79d96d4462ad6509b2b05a4c85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:29,586 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:29,586 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/C, priority=13, startTime=1731525928736; duration=0sec 2024-11-13T19:25:29,586 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:29,586 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:C 2024-11-13T19:25:29,644 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/8348ef5e68474188b333eb3b6117757a 2024-11-13T19:25:29,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/9d949cdfc1d64eeb8697f87e5ef46c69 is 50, key is test_row_0/C:col10/1731525927537/Put/seqid=0 2024-11-13T19:25:29,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742276_1452 (size=12151) 2024-11-13T19:25:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:29,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:29,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525989718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525989719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525989723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:29,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525989723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,726 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,727 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525989831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525989831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525989831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525989831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:29,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525990042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525990042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525990042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,047 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525990042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,054 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/9d949cdfc1d64eeb8697f87e5ef46c69 2024-11-13T19:25:30,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/abe98e1c48cd45c7934cf6b3d3dd7357 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/abe98e1c48cd45c7934cf6b3d3dd7357 2024-11-13T19:25:30,061 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/abe98e1c48cd45c7934cf6b3d3dd7357, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T19:25:30,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/8348ef5e68474188b333eb3b6117757a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/8348ef5e68474188b333eb3b6117757a 2024-11-13T19:25:30,065 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/8348ef5e68474188b333eb3b6117757a, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T19:25:30,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/9d949cdfc1d64eeb8697f87e5ef46c69 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9d949cdfc1d64eeb8697f87e5ef46c69 2024-11-13T19:25:30,068 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9d949cdfc1d64eeb8697f87e5ef46c69, entries=150, sequenceid=191, filesize=11.9 K 2024-11-13T19:25:30,069 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d17e3cc0ba121595297bf675d9f1d2e3 in 1242ms, sequenceid=191, compaction requested=false 2024-11-13T19:25:30,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:30,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-13T19:25:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-13T19:25:30,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-13T19:25:30,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-13T19:25:30,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0100 sec 2024-11-13T19:25:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-13T19:25:30,212 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:30,212 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-13T19:25:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:25:30,214 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:30,214 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:30,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:30,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:25:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:30,354 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:30,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T19:25:30,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/7a5f1f3e87af4ddb94e08e4b58933a36 is 50, key is test_row_0/A:col10/1731525929721/Put/seqid=0 2024-11-13T19:25:30,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742277_1453 (size=14541) 2024-11-13T19:25:30,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:30,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:30,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525990389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525990389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525990394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525990396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525990501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525990501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,509 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525990512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525990512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,517 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,518 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:30,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:30,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:25:30,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:30,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:30,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525990711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525990712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525990722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525990722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:30,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:30,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/7a5f1f3e87af4ddb94e08e4b58933a36 2024-11-13T19:25:30,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/44379dc170ff4f078f3f99525935cc99 is 50, key is test_row_0/B:col10/1731525929721/Put/seqid=0 2024-11-13T19:25:30,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742278_1454 (size=12151) 2024-11-13T19:25:30,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:30,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:25:30,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:30,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:30,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525991021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525991021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525991032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525991032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,037 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:31,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:31,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/44379dc170ff4f078f3f99525935cc99 2024-11-13T19:25:31,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/13fd99a9cbb7449a8d69d64af872d595 is 50, key is test_row_0/C:col10/1731525929721/Put/seqid=0 2024-11-13T19:25:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742279_1455 (size=12151) 2024-11-13T19:25:31,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:31,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:31,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:25:31,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:31,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:31,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525991531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525991532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,538 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525991542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525991542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:31,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:31,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/13fd99a9cbb7449a8d69d64af872d595 2024-11-13T19:25:31,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/7a5f1f3e87af4ddb94e08e4b58933a36 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7a5f1f3e87af4ddb94e08e4b58933a36 2024-11-13T19:25:31,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7a5f1f3e87af4ddb94e08e4b58933a36, entries=200, sequenceid=208, filesize=14.2 K 2024-11-13T19:25:31,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/44379dc170ff4f078f3f99525935cc99 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/44379dc170ff4f078f3f99525935cc99 2024-11-13T19:25:31,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:31,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:31,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,588 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:31,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/44379dc170ff4f078f3f99525935cc99, entries=150, sequenceid=208, filesize=11.9 K 2024-11-13T19:25:31,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/13fd99a9cbb7449a8d69d64af872d595 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/13fd99a9cbb7449a8d69d64af872d595 2024-11-13T19:25:31,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/13fd99a9cbb7449a8d69d64af872d595, entries=150, sequenceid=208, filesize=11.9 K 2024-11-13T19:25:31,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d17e3cc0ba121595297bf675d9f1d2e3 in 1240ms, sequenceid=208, compaction requested=true 2024-11-13T19:25:31,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:31,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:31,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:31,595 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:31,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:31,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:31,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:31,595 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:31,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:31,595 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:31,595 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:31,595 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/B is initiating minor compaction (all files) 2024-11-13T19:25:31,595 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/A is initiating minor compaction (all files) 2024-11-13T19:25:31,596 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/B in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,596 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/A in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,596 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/5465fa9111a244f682af34075f557dd7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/8348ef5e68474188b333eb3b6117757a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/44379dc170ff4f078f3f99525935cc99] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=36.0 K 2024-11-13T19:25:31,596 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/cc9f5838accb4e0591b396d580e3c075, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/abe98e1c48cd45c7934cf6b3d3dd7357, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7a5f1f3e87af4ddb94e08e4b58933a36] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=38.3 K 2024-11-13T19:25:31,596 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc9f5838accb4e0591b396d580e3c075, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731525926304 2024-11-13T19:25:31,596 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5465fa9111a244f682af34075f557dd7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731525926304 2024-11-13T19:25:31,596 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting abe98e1c48cd45c7934cf6b3d3dd7357, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731525927532 2024-11-13T19:25:31,596 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 8348ef5e68474188b333eb3b6117757a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731525927532 2024-11-13T19:25:31,596 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a5f1f3e87af4ddb94e08e4b58933a36, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731525929715 2024-11-13T19:25:31,596 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 44379dc170ff4f078f3f99525935cc99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731525929721 2024-11-13T19:25:31,601 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#A#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:31,601 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:31,601 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/948395e63bcc4e5c9b026189804918d0 is 50, key is test_row_0/B:col10/1731525929721/Put/seqid=0 2024-11-13T19:25:31,601 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/19c20512f42f4e42bcfe4a3ba242e448 is 50, key is test_row_0/A:col10/1731525929721/Put/seqid=0 2024-11-13T19:25:31,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742280_1456 (size=12663) 2024-11-13T19:25:31,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742281_1457 (size=12663) 2024-11-13T19:25:31,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-13T19:25:31,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:31,741 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T19:25:31,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/05853ea093c44aee9c425b1918421167 is 50, key is test_row_0/A:col10/1731525930395/Put/seqid=0 2024-11-13T19:25:31,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742282_1458 (size=12151) 2024-11-13T19:25:31,747 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/05853ea093c44aee9c425b1918421167 2024-11-13T19:25:31,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/95c82e6c194f4b70828766a623f8b0b0 is 50, key is test_row_0/B:col10/1731525930395/Put/seqid=0 2024-11-13T19:25:31,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742283_1459 (size=12151) 2024-11-13T19:25:32,008 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/948395e63bcc4e5c9b026189804918d0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/948395e63bcc4e5c9b026189804918d0 2024-11-13T19:25:32,008 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/19c20512f42f4e42bcfe4a3ba242e448 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/19c20512f42f4e42bcfe4a3ba242e448 2024-11-13T19:25:32,013 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/B of d17e3cc0ba121595297bf675d9f1d2e3 into 948395e63bcc4e5c9b026189804918d0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:32,013 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/A of d17e3cc0ba121595297bf675d9f1d2e3 into 19c20512f42f4e42bcfe4a3ba242e448(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:32,013 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/A, priority=13, startTime=1731525931595; duration=0sec 2024-11-13T19:25:32,013 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/B, priority=13, startTime=1731525931595; duration=0sec 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:A 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:B 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:32,013 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/C is initiating minor compaction (all files) 2024-11-13T19:25:32,013 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/C in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:32,014 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a5fa7a79d96d4462ad6509b2b05a4c85, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9d949cdfc1d64eeb8697f87e5ef46c69, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/13fd99a9cbb7449a8d69d64af872d595] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=36.0 K 2024-11-13T19:25:32,014 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5fa7a79d96d4462ad6509b2b05a4c85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731525926304 2024-11-13T19:25:32,014 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d949cdfc1d64eeb8697f87e5ef46c69, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731525927532 2024-11-13T19:25:32,014 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13fd99a9cbb7449a8d69d64af872d595, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731525929721 2024-11-13T19:25:32,020 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#C#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:32,020 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/633bed984575485abdae823e7079c18f is 50, key is test_row_0/C:col10/1731525929721/Put/seqid=0 2024-11-13T19:25:32,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742284_1460 (size=12663) 2024-11-13T19:25:32,156 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/95c82e6c194f4b70828766a623f8b0b0 2024-11-13T19:25:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/a56c1240f1764005aed0eed10e2add12 is 50, key is test_row_0/C:col10/1731525930395/Put/seqid=0 2024-11-13T19:25:32,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742285_1461 (size=12151) 2024-11-13T19:25:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:25:32,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:32,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:32,433 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/633bed984575485abdae823e7079c18f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/633bed984575485abdae823e7079c18f 2024-11-13T19:25:32,440 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/C of d17e3cc0ba121595297bf675d9f1d2e3 into 633bed984575485abdae823e7079c18f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:32,440 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:32,440 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/C, priority=13, startTime=1731525931595; duration=0sec 2024-11-13T19:25:32,440 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:32,440 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:C 2024-11-13T19:25:32,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:32,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525992485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:32,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525992551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:32,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525992552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:32,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:32,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525992552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:32,555 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:32,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525992552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,555 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:32,556 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:32,556 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,557 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:32,567 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/a56c1240f1764005aed0eed10e2add12 2024-11-13T19:25:32,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/05853ea093c44aee9c425b1918421167 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/05853ea093c44aee9c425b1918421167 2024-11-13T19:25:32,573 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/05853ea093c44aee9c425b1918421167, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T19:25:32,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/95c82e6c194f4b70828766a623f8b0b0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/95c82e6c194f4b70828766a623f8b0b0 2024-11-13T19:25:32,577 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/95c82e6c194f4b70828766a623f8b0b0, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T19:25:32,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/a56c1240f1764005aed0eed10e2add12 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a56c1240f1764005aed0eed10e2add12 2024-11-13T19:25:32,580 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a56c1240f1764005aed0eed10e2add12, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T19:25:32,581 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d17e3cc0ba121595297bf675d9f1d2e3 in 840ms, sequenceid=230, compaction requested=false 2024-11-13T19:25:32,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:32,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:32,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-13T19:25:32,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-13T19:25:32,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-13T19:25:32,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3690 sec 2024-11-13T19:25:32,586 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.3730 sec 2024-11-13T19:25:32,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:32,604 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:32,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T19:25:32,608 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/c44fac2e236d418aa61265f499dbd626 is 50, key is test_row_0/A:col10/1731525932464/Put/seqid=0 2024-11-13T19:25:32,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742286_1462 (size=14541) 2024-11-13T19:25:32,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/c44fac2e236d418aa61265f499dbd626 2024-11-13T19:25:32,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/c2f390182e6a4f0e91eccda733be8572 is 50, key is test_row_0/B:col10/1731525932464/Put/seqid=0 2024-11-13T19:25:32,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742287_1463 (size=12151) 2024-11-13T19:25:32,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:32,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525992736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:32,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:32,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:32,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525992851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:32,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:33,032 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/c2f390182e6a4f0e91eccda733be8572 2024-11-13T19:25:33,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/dbcd3928ee70451f99b37dd490199059 is 50, key is test_row_0/C:col10/1731525932464/Put/seqid=0 2024-11-13T19:25:33,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742288_1464 (size=12151) 2024-11-13T19:25:33,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:33,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525993061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:33,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:33,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525993382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:33,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:33,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:33,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:33,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/dbcd3928ee70451f99b37dd490199059 2024-11-13T19:25:33,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/c44fac2e236d418aa61265f499dbd626 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/c44fac2e236d418aa61265f499dbd626 2024-11-13T19:25:33,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/c44fac2e236d418aa61265f499dbd626, entries=200, sequenceid=248, filesize=14.2 K 2024-11-13T19:25:33,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/c2f390182e6a4f0e91eccda733be8572 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/c2f390182e6a4f0e91eccda733be8572 2024-11-13T19:25:33,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/c2f390182e6a4f0e91eccda733be8572, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T19:25:33,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/dbcd3928ee70451f99b37dd490199059 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/dbcd3928ee70451f99b37dd490199059 2024-11-13T19:25:33,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/dbcd3928ee70451f99b37dd490199059, entries=150, sequenceid=248, filesize=11.9 K 2024-11-13T19:25:33,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for d17e3cc0ba121595297bf675d9f1d2e3 in 850ms, sequenceid=248, compaction requested=true 2024-11-13T19:25:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:33,455 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:33,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:33,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:33,455 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:33,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:33,455 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/B is initiating minor compaction (all files) 2024-11-13T19:25:33,455 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/A is initiating minor compaction (all files) 2024-11-13T19:25:33,456 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/A in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:33,456 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/B in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:33,456 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/19c20512f42f4e42bcfe4a3ba242e448, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/05853ea093c44aee9c425b1918421167, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/c44fac2e236d418aa61265f499dbd626] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=38.4 K 2024-11-13T19:25:33,456 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/948395e63bcc4e5c9b026189804918d0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/95c82e6c194f4b70828766a623f8b0b0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/c2f390182e6a4f0e91eccda733be8572] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=36.1 K 2024-11-13T19:25:33,456 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 948395e63bcc4e5c9b026189804918d0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731525929721 2024-11-13T19:25:33,456 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19c20512f42f4e42bcfe4a3ba242e448, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731525929721 2024-11-13T19:25:33,456 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 95c82e6c194f4b70828766a623f8b0b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525930375 2024-11-13T19:25:33,456 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05853ea093c44aee9c425b1918421167, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525930375 2024-11-13T19:25:33,456 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c2f390182e6a4f0e91eccda733be8572, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731525932464 2024-11-13T19:25:33,456 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting c44fac2e236d418aa61265f499dbd626, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731525932464 2024-11-13T19:25:33,472 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#B#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:33,472 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:33,472 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/2e61fbebc6844e31b75141e2f696a536 is 50, key is test_row_0/A:col10/1731525932464/Put/seqid=0 2024-11-13T19:25:33,472 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/ee3ae67a129f4436a4b44c3195cce511 is 50, key is test_row_0/B:col10/1731525932464/Put/seqid=0 2024-11-13T19:25:33,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742290_1466 (size=12765) 2024-11-13T19:25:33,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742289_1465 (size=12765) 2024-11-13T19:25:33,898 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/ee3ae67a129f4436a4b44c3195cce511 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ee3ae67a129f4436a4b44c3195cce511 2024-11-13T19:25:33,898 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/2e61fbebc6844e31b75141e2f696a536 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2e61fbebc6844e31b75141e2f696a536 2024-11-13T19:25:33,902 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/A of d17e3cc0ba121595297bf675d9f1d2e3 into 2e61fbebc6844e31b75141e2f696a536(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:33,902 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/B of d17e3cc0ba121595297bf675d9f1d2e3 into ee3ae67a129f4436a4b44c3195cce511(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:33,902 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/A, priority=13, startTime=1731525933455; duration=0sec 2024-11-13T19:25:33,902 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/B, priority=13, startTime=1731525933455; duration=0sec 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:A 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:B 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:33,902 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/C is initiating minor compaction (all files) 2024-11-13T19:25:33,902 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/C in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:33,903 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/633bed984575485abdae823e7079c18f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a56c1240f1764005aed0eed10e2add12, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/dbcd3928ee70451f99b37dd490199059] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=36.1 K 2024-11-13T19:25:33,903 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 633bed984575485abdae823e7079c18f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731525929721 2024-11-13T19:25:33,903 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a56c1240f1764005aed0eed10e2add12, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731525930375 2024-11-13T19:25:33,903 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbcd3928ee70451f99b37dd490199059, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731525932464 2024-11-13T19:25:33,908 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:33,908 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/1d7ae2f67d73415f87bc821ee72154dc is 50, key is test_row_0/C:col10/1731525932464/Put/seqid=0 2024-11-13T19:25:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:33,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:33,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-13T19:25:33,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742291_1467 (size=12765) 2024-11-13T19:25:33,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/d3f568552df24deebdc8e6201798d7cd is 50, key is test_row_0/A:col10/1731525933908/Put/seqid=0 2024-11-13T19:25:33,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742292_1468 (size=14741) 2024-11-13T19:25:33,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:33,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525993963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:33,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525994072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:34,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,078 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525994281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,315 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/1d7ae2f67d73415f87bc821ee72154dc as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1d7ae2f67d73415f87bc821ee72154dc 2024-11-13T19:25:34,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/d3f568552df24deebdc8e6201798d7cd 2024-11-13T19:25:34,319 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/C of d17e3cc0ba121595297bf675d9f1d2e3 into 1d7ae2f67d73415f87bc821ee72154dc(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:34,319 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:34,319 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/C, priority=13, startTime=1731525933455; duration=0sec 2024-11-13T19:25:34,319 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:34,319 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:C 2024-11-13T19:25:34,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/15217424ec7a417ca6a096cb7a1f1b9b is 50, key is test_row_0/B:col10/1731525933908/Put/seqid=0 2024-11-13T19:25:34,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742293_1469 (size=12301) 2024-11-13T19:25:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-13T19:25:34,382 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:34,382 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-13T19:25:34,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:25:34,383 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:34,384 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:34,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:34,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:25:34,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:25:34,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:34,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46482 deadline: 1731525994561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:34,565 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_0' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,566 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46404 deadline: 1731525994571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46472 deadline: 1731525994571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:34,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46418 deadline: 1731525994571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:34,578 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,578 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_2' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,578 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to e4b56bc8c4b3,40371,1731525792389 for 'test_row_1' in d17e3cc0ba121595297bf675d9f1d2e3 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-13T19:25:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525994592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:34,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:34,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:25:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:25:34,731 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/15217424ec7a417ca6a096cb7a1f1b9b 2024-11-13T19:25:34,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/20c8cf3c0c2d4697b58b856e8e05ffe9 is 50, key is test_row_0/C:col10/1731525933908/Put/seqid=0 2024-11-13T19:25:34,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742294_1470 (size=12301) 2024-11-13T19:25:34,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:25:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:25:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:34,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:25:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525995102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:35,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/20c8cf3c0c2d4697b58b856e8e05ffe9 2024-11-13T19:25:35,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/d3f568552df24deebdc8e6201798d7cd as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d3f568552df24deebdc8e6201798d7cd 2024-11-13T19:25:35,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d3f568552df24deebdc8e6201798d7cd, entries=200, sequenceid=272, filesize=14.4 K 2024-11-13T19:25:35,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/15217424ec7a417ca6a096cb7a1f1b9b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/15217424ec7a417ca6a096cb7a1f1b9b 2024-11-13T19:25:35,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:25:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:35,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:35,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:35,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/15217424ec7a417ca6a096cb7a1f1b9b, entries=150, sequenceid=272, filesize=12.0 K 2024-11-13T19:25:35,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/20c8cf3c0c2d4697b58b856e8e05ffe9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/20c8cf3c0c2d4697b58b856e8e05ffe9 2024-11-13T19:25:35,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/20c8cf3c0c2d4697b58b856e8e05ffe9, entries=150, sequenceid=272, filesize=12.0 K 2024-11-13T19:25:35,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for d17e3cc0ba121595297bf675d9f1d2e3 in 1246ms, sequenceid=272, compaction requested=false 2024-11-13T19:25:35,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:35,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-13T19:25:35,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:35,300 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T19:25:35,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/76d020c48617488c803478150fd321b7 is 50, key is test_row_0/A:col10/1731525933950/Put/seqid=0 2024-11-13T19:25:35,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742295_1471 (size=12301) 2024-11-13T19:25:35,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:25:35,709 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/76d020c48617488c803478150fd321b7 2024-11-13T19:25:35,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/7904e9afc72b4cba9ddd9b1b0f607975 is 50, key is test_row_0/B:col10/1731525933950/Put/seqid=0 2024-11-13T19:25:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742296_1472 (size=12301) 2024-11-13T19:25:36,169 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/7904e9afc72b4cba9ddd9b1b0f607975 2024-11-13T19:25:36,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. as already flushing 2024-11-13T19:25:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:36,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/ac724dd220cd4b0fb7baa4e1da4a0929 is 50, key is test_row_0/C:col10/1731525933950/Put/seqid=0 2024-11-13T19:25:36,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742297_1473 (size=12301) 2024-11-13T19:25:36,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525996284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:36,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525996402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,407 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:25:36,580 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=287 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/ac724dd220cd4b0fb7baa4e1da4a0929 2024-11-13T19:25:36,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/76d020c48617488c803478150fd321b7 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/76d020c48617488c803478150fd321b7 2024-11-13T19:25:36,586 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/76d020c48617488c803478150fd321b7, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:25:36,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/7904e9afc72b4cba9ddd9b1b0f607975 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7904e9afc72b4cba9ddd9b1b0f607975 2024-11-13T19:25:36,590 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7904e9afc72b4cba9ddd9b1b0f607975, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:25:36,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/ac724dd220cd4b0fb7baa4e1da4a0929 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/ac724dd220cd4b0fb7baa4e1da4a0929 2024-11-13T19:25:36,593 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/ac724dd220cd4b0fb7baa4e1da4a0929, entries=150, sequenceid=287, filesize=12.0 K 2024-11-13T19:25:36,594 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d17e3cc0ba121595297bf675d9f1d2e3 in 1294ms, sequenceid=287, compaction requested=true 2024-11-13T19:25:36,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:36,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:36,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-13T19:25:36,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-13T19:25:36,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-13T19:25:36,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2110 sec 2024-11-13T19:25:36,598 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 2.2150 sec 2024-11-13T19:25:36,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:36,613 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:36,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-13T19:25:36,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/8aefb25f66914063b69c89f2a5b3fdf0 is 50, key is test_row_0/A:col10/1731525936282/Put/seqid=0 2024-11-13T19:25:36,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742298_1474 (size=14741) 2024-11-13T19:25:36,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525996676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:36,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525996792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:37,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:37,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525997001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:37,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/8aefb25f66914063b69c89f2a5b3fdf0 2024-11-13T19:25:37,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/3d1a92a9e6aa45258c69e01cc62c5114 is 50, key is test_row_0/B:col10/1731525936282/Put/seqid=0 2024-11-13T19:25:37,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742299_1475 (size=12301) 2024-11-13T19:25:37,286 INFO [Thread-1876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1876. 2024-11-13T19:25:37,286 DEBUG [Thread-1876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:37,286 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,286 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:37,286 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-13T19:25:37,286 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:37,286 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,286 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:37,291 INFO [Thread-1872 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1872. 2024-11-13T19:25:37,291 DEBUG [Thread-1872 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:37,291 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,291 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:37,294 INFO [Thread-1868 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1868. 2024-11-13T19:25:37,294 DEBUG [Thread-1868 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:37,294 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,295 DEBUG [Thread-1868 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:37,295 INFO [Thread-1874 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1874. 2024-11-13T19:25:37,295 DEBUG [Thread-1874 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:37,295 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,295 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:37,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:37,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:37,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525997312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:37,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:37,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/3d1a92a9e6aa45258c69e01cc62c5114 2024-11-13T19:25:37,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/b389f4c44b5f43d68194aab0a2fed5e2 is 50, key is test_row_0/C:col10/1731525936282/Put/seqid=0 2024-11-13T19:25:37,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742300_1476 (size=12301) 2024-11-13T19:25:37,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46448 deadline: 1731525997822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 , the old value is region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:37,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=2 because the exception is null or not the one we care about 2024-11-13T19:25:37,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/b389f4c44b5f43d68194aab0a2fed5e2 2024-11-13T19:25:37,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/8aefb25f66914063b69c89f2a5b3fdf0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/8aefb25f66914063b69c89f2a5b3fdf0 2024-11-13T19:25:37,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/8aefb25f66914063b69c89f2a5b3fdf0, entries=200, sequenceid=310, filesize=14.4 K 2024-11-13T19:25:37,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/3d1a92a9e6aa45258c69e01cc62c5114 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/3d1a92a9e6aa45258c69e01cc62c5114 2024-11-13T19:25:37,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/3d1a92a9e6aa45258c69e01cc62c5114, entries=150, sequenceid=310, filesize=12.0 K 2024-11-13T19:25:37,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/b389f4c44b5f43d68194aab0a2fed5e2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b389f4c44b5f43d68194aab0a2fed5e2 2024-11-13T19:25:37,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b389f4c44b5f43d68194aab0a2fed5e2, entries=150, sequenceid=310, filesize=12.0 K 2024-11-13T19:25:37,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d17e3cc0ba121595297bf675d9f1d2e3 in 1256ms, sequenceid=310, compaction requested=true 2024-11-13T19:25:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:37,869 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:25:37,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d17e3cc0ba121595297bf675d9f1d2e3:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:37,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:37,870 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:25:37,871 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:25:37,871 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:25:37,871 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/A is initiating minor compaction (all files) 2024-11-13T19:25:37,871 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/B is initiating minor compaction (all files) 2024-11-13T19:25:37,871 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/A in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:37,871 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/B in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:37,871 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ee3ae67a129f4436a4b44c3195cce511, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/15217424ec7a417ca6a096cb7a1f1b9b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7904e9afc72b4cba9ddd9b1b0f607975, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/3d1a92a9e6aa45258c69e01cc62c5114] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=48.5 K 2024-11-13T19:25:37,871 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2e61fbebc6844e31b75141e2f696a536, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d3f568552df24deebdc8e6201798d7cd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/76d020c48617488c803478150fd321b7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/8aefb25f66914063b69c89f2a5b3fdf0] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=53.3 K 2024-11-13T19:25:37,872 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3ae67a129f4436a4b44c3195cce511, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731525932464 2024-11-13T19:25:37,872 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e61fbebc6844e31b75141e2f696a536, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731525932464 2024-11-13T19:25:37,872 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 15217424ec7a417ca6a096cb7a1f1b9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731525932728 2024-11-13T19:25:37,872 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f568552df24deebdc8e6201798d7cd, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731525932719 2024-11-13T19:25:37,872 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7904e9afc72b4cba9ddd9b1b0f607975, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731525933937 2024-11-13T19:25:37,872 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76d020c48617488c803478150fd321b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731525933937 2024-11-13T19:25:37,873 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d1a92a9e6aa45258c69e01cc62c5114, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731525936251 2024-11-13T19:25:37,873 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aefb25f66914063b69c89f2a5b3fdf0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731525936251 2024-11-13T19:25:37,884 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#B#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:37,884 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#A#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:37,884 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/05325bcf05b04104a2bb070d6b0bee4a is 50, key is test_row_0/B:col10/1731525936282/Put/seqid=0 2024-11-13T19:25:37,884 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/655e48f5b04f4182bdb9a735fed6ddb1 is 50, key is test_row_0/A:col10/1731525936282/Put/seqid=0 2024-11-13T19:25:37,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742301_1477 (size=13051) 2024-11-13T19:25:37,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742302_1478 (size=13051) 2024-11-13T19:25:38,300 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/655e48f5b04f4182bdb9a735fed6ddb1 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/655e48f5b04f4182bdb9a735fed6ddb1 2024-11-13T19:25:38,301 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/05325bcf05b04104a2bb070d6b0bee4a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/05325bcf05b04104a2bb070d6b0bee4a 2024-11-13T19:25:38,306 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/A of d17e3cc0ba121595297bf675d9f1d2e3 into 655e48f5b04f4182bdb9a735fed6ddb1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:38,306 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:38,306 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/A, priority=12, startTime=1731525937869; duration=0sec 2024-11-13T19:25:38,306 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:38,306 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:A 2024-11-13T19:25:38,306 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T19:25:38,306 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/B of d17e3cc0ba121595297bf675d9f1d2e3 into 05325bcf05b04104a2bb070d6b0bee4a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:38,306 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:38,306 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/B, priority=12, startTime=1731525937869; duration=0sec 2024-11-13T19:25:38,306 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:38,306 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:B 2024-11-13T19:25:38,307 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-13T19:25:38,307 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): d17e3cc0ba121595297bf675d9f1d2e3/C is initiating minor compaction (all files) 2024-11-13T19:25:38,307 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d17e3cc0ba121595297bf675d9f1d2e3/C in TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:38,308 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1d7ae2f67d73415f87bc821ee72154dc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/20c8cf3c0c2d4697b58b856e8e05ffe9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/ac724dd220cd4b0fb7baa4e1da4a0929, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b389f4c44b5f43d68194aab0a2fed5e2] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp, totalSize=48.5 K 2024-11-13T19:25:38,308 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d7ae2f67d73415f87bc821ee72154dc, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731525932464 2024-11-13T19:25:38,308 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c8cf3c0c2d4697b58b856e8e05ffe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731525932728 2024-11-13T19:25:38,309 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac724dd220cd4b0fb7baa4e1da4a0929, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=287, earliestPutTs=1731525933937 2024-11-13T19:25:38,309 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b389f4c44b5f43d68194aab0a2fed5e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731525936251 2024-11-13T19:25:38,318 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d17e3cc0ba121595297bf675d9f1d2e3#C#compaction#410 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:38,319 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/650e0abdd33e4e2687111061b4bccb27 is 50, key is test_row_0/C:col10/1731525936282/Put/seqid=0 2024-11-13T19:25:38,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742303_1479 (size=13051) 2024-11-13T19:25:38,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-13T19:25:38,553 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:38,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:38,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T19:25:38,585 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-11-13T19:25:38,585 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:38,585 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,585 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:38,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/690a438b8d174a5b88cfcd9d2fe34a56 is 50, key is test_row_0/A:col10/1731525938582/Put/seqid=0 2024-11-13T19:25:38,594 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-13T19:25:38,594 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-13T19:25:38,594 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:38,594 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:38,594 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,594 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,594 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,594 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:38,594 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:38,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742304_1480 (size=12301) 2024-11-13T19:25:38,622 INFO [Thread-1857 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1857. 2024-11-13T19:25:38,622 DEBUG [Thread-1857 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:38,622 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,622 DEBUG [Thread-1857 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:38,736 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/650e0abdd33e4e2687111061b4bccb27 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/650e0abdd33e4e2687111061b4bccb27 2024-11-13T19:25:38,740 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d17e3cc0ba121595297bf675d9f1d2e3/C of d17e3cc0ba121595297bf675d9f1d2e3 into 650e0abdd33e4e2687111061b4bccb27(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:38,740 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:38,740 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., storeName=d17e3cc0ba121595297bf675d9f1d2e3/C, priority=12, startTime=1731525937870; duration=0sec 2024-11-13T19:25:38,740 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:38,740 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d17e3cc0ba121595297bf675d9f1d2e3:C 2024-11-13T19:25:38,845 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-11-13T19:25:38,845 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:25:38,845 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,845 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,846 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1664 2024-11-13T19:25:38,846 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4990 rows 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1654 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4962 rows 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1662 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4981 rows 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1663 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4987 rows 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1658 2024-11-13T19:25:38,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4974 rows 2024-11-13T19:25:38,847 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:25:38,847 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:25:38,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:38,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:25:38,849 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:25:38,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:38,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:25:38,854 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525938853"}]},"ts":"1731525938853"} 2024-11-13T19:25:38,855 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:25:38,855 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:25:38,856 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:25:38,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, UNASSIGN}] 2024-11-13T19:25:38,859 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, UNASSIGN 2024-11-13T19:25:38,859 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d17e3cc0ba121595297bf675d9f1d2e3, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:38,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, UNASSIGN because future has completed 2024-11-13T19:25:38,861 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:25:38,861 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:25:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:25:38,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/690a438b8d174a5b88cfcd9d2fe34a56 2024-11-13T19:25:39,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/aa87a9bc6b574a40ba461769cfd7036c is 50, key is test_row_0/B:col10/1731525938582/Put/seqid=0 2024-11-13T19:25:39,014 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(122): Close d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:39,014 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:25:39,014 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1722): Closing d17e3cc0ba121595297bf675d9f1d2e3, disabling compactions & flushes 2024-11-13T19:25:39,014 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:39,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742305_1481 (size=12301) 2024-11-13T19:25:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:25:39,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/aa87a9bc6b574a40ba461769cfd7036c 2024-11-13T19:25:39,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/85a59f323f534b22881cb2c76487e583 is 50, key is test_row_0/C:col10/1731525938582/Put/seqid=0 2024-11-13T19:25:39,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742306_1482 (size=12301) 2024-11-13T19:25:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:25:39,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/85a59f323f534b22881cb2c76487e583 2024-11-13T19:25:39,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/690a438b8d174a5b88cfcd9d2fe34a56 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/690a438b8d174a5b88cfcd9d2fe34a56 2024-11-13T19:25:39,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/690a438b8d174a5b88cfcd9d2fe34a56, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T19:25:39,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/aa87a9bc6b574a40ba461769cfd7036c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/aa87a9bc6b574a40ba461769cfd7036c 2024-11-13T19:25:39,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/aa87a9bc6b574a40ba461769cfd7036c, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T19:25:39,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/85a59f323f534b22881cb2c76487e583 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/85a59f323f534b22881cb2c76487e583 2024-11-13T19:25:39,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/85a59f323f534b22881cb2c76487e583, entries=150, sequenceid=326, filesize=12.0 K 2024-11-13T19:25:39,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=26.84 KB/27480 for d17e3cc0ba121595297bf675d9f1d2e3 in 1275ms, sequenceid=326, compaction requested=false 2024-11-13T19:25:39,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d17e3cc0ba121595297bf675d9f1d2e3: 2024-11-13T19:25:39,859 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:39,859 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:39,859 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. after waiting 0 ms 2024-11-13T19:25:39,859 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:39,859 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(2902): Flushing d17e3cc0ba121595297bf675d9f1d2e3 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-13T19:25:39,864 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/4ba51273ab434a9587b4d5913efa7e56 is 50, key is test_row_0/A:col10/1731525938843/Put/seqid=0 2024-11-13T19:25:39,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742307_1483 (size=12301) 2024-11-13T19:25:39,869 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/4ba51273ab434a9587b4d5913efa7e56 2024-11-13T19:25:39,878 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/344be036a7694e86ad8588d39da1c75c is 50, key is test_row_0/B:col10/1731525938843/Put/seqid=0 2024-11-13T19:25:39,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742308_1484 (size=12301) 2024-11-13T19:25:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:25:40,282 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/344be036a7694e86ad8588d39da1c75c 2024-11-13T19:25:40,291 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/f3817affdc79485fa055a21f869a4957 is 50, key is test_row_0/C:col10/1731525938843/Put/seqid=0 2024-11-13T19:25:40,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742309_1485 (size=12301) 2024-11-13T19:25:40,698 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/f3817affdc79485fa055a21f869a4957 2024-11-13T19:25:40,707 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/A/4ba51273ab434a9587b4d5913efa7e56 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/4ba51273ab434a9587b4d5913efa7e56 2024-11-13T19:25:40,710 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/4ba51273ab434a9587b4d5913efa7e56, entries=150, sequenceid=334, filesize=12.0 K 2024-11-13T19:25:40,711 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/B/344be036a7694e86ad8588d39da1c75c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/344be036a7694e86ad8588d39da1c75c 2024-11-13T19:25:40,714 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/344be036a7694e86ad8588d39da1c75c, entries=150, sequenceid=334, filesize=12.0 K 2024-11-13T19:25:40,715 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/.tmp/C/f3817affdc79485fa055a21f869a4957 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f3817affdc79485fa055a21f869a4957 2024-11-13T19:25:40,717 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f3817affdc79485fa055a21f869a4957, entries=150, sequenceid=334, filesize=12.0 K 2024-11-13T19:25:40,718 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d17e3cc0ba121595297bf675d9f1d2e3 in 859ms, sequenceid=334, compaction requested=true 2024-11-13T19:25:40,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2dbda7a5f6034509a7cc274660e371bf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/9c697579b89c4f05bf13b2dce93597fa, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/322905bb75d54290afd1c173e059eaf0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/95eca11188dd42ab89e5246f0aac513c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/236cc9a5db4841cb8adcfe9900f4e6a2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7b6984d22b12405fa7712649998cb1b7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/e46a184f569546b4992b42dd97eb6404, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/0c291b2853154cb8a609651991643258, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/384e05afd5d74a22933c6baed0f552ea, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d1e23127514f42e59c50adb9c5bc06b6, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/bd24e589f82440cfa1896b686e9f4f0c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/cc9f5838accb4e0591b396d580e3c075, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/abe98e1c48cd45c7934cf6b3d3dd7357, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7a5f1f3e87af4ddb94e08e4b58933a36, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/19c20512f42f4e42bcfe4a3ba242e448, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/05853ea093c44aee9c425b1918421167, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/c44fac2e236d418aa61265f499dbd626, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2e61fbebc6844e31b75141e2f696a536, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d3f568552df24deebdc8e6201798d7cd, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/76d020c48617488c803478150fd321b7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/8aefb25f66914063b69c89f2a5b3fdf0] to archive 2024-11-13T19:25:40,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:25:40,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2dbda7a5f6034509a7cc274660e371bf to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2dbda7a5f6034509a7cc274660e371bf 2024-11-13T19:25:40,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/9c697579b89c4f05bf13b2dce93597fa to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/9c697579b89c4f05bf13b2dce93597fa 2024-11-13T19:25:40,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/322905bb75d54290afd1c173e059eaf0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/322905bb75d54290afd1c173e059eaf0 2024-11-13T19:25:40,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/95eca11188dd42ab89e5246f0aac513c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/95eca11188dd42ab89e5246f0aac513c 2024-11-13T19:25:40,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/236cc9a5db4841cb8adcfe9900f4e6a2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/236cc9a5db4841cb8adcfe9900f4e6a2 2024-11-13T19:25:40,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7b6984d22b12405fa7712649998cb1b7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7b6984d22b12405fa7712649998cb1b7 2024-11-13T19:25:40,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/e46a184f569546b4992b42dd97eb6404 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/e46a184f569546b4992b42dd97eb6404 2024-11-13T19:25:40,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/fe4e35e1bf4c49c2ad7c4bb38c6e5a4a 2024-11-13T19:25:40,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/0c291b2853154cb8a609651991643258 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/0c291b2853154cb8a609651991643258 2024-11-13T19:25:40,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/384e05afd5d74a22933c6baed0f552ea to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/384e05afd5d74a22933c6baed0f552ea 2024-11-13T19:25:40,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d1e23127514f42e59c50adb9c5bc06b6 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d1e23127514f42e59c50adb9c5bc06b6 2024-11-13T19:25:40,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/bd24e589f82440cfa1896b686e9f4f0c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/bd24e589f82440cfa1896b686e9f4f0c 2024-11-13T19:25:40,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/cc9f5838accb4e0591b396d580e3c075 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/cc9f5838accb4e0591b396d580e3c075 2024-11-13T19:25:40,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/abe98e1c48cd45c7934cf6b3d3dd7357 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/abe98e1c48cd45c7934cf6b3d3dd7357 2024-11-13T19:25:40,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7a5f1f3e87af4ddb94e08e4b58933a36 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/7a5f1f3e87af4ddb94e08e4b58933a36 2024-11-13T19:25:40,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/19c20512f42f4e42bcfe4a3ba242e448 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/19c20512f42f4e42bcfe4a3ba242e448 2024-11-13T19:25:40,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/05853ea093c44aee9c425b1918421167 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/05853ea093c44aee9c425b1918421167 2024-11-13T19:25:40,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/c44fac2e236d418aa61265f499dbd626 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/c44fac2e236d418aa61265f499dbd626 2024-11-13T19:25:40,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2e61fbebc6844e31b75141e2f696a536 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/2e61fbebc6844e31b75141e2f696a536 2024-11-13T19:25:40,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d3f568552df24deebdc8e6201798d7cd to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/d3f568552df24deebdc8e6201798d7cd 2024-11-13T19:25:40,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/76d020c48617488c803478150fd321b7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/76d020c48617488c803478150fd321b7 2024-11-13T19:25:40,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/8aefb25f66914063b69c89f2a5b3fdf0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/8aefb25f66914063b69c89f2a5b3fdf0 2024-11-13T19:25:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/18b86c323e5442d0af0beee8108dc22d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/656aaec0ef8b4d679d118b2d0a21d53a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7fcf72c77d9e469487cd645a14d537af, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/dcb3f5306bd14e969b1ed35d25e95ca2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/cc0a931ac70642b991ddf6dc3746e637, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ce88e46ad93e49769b64fb6e2aa99645, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a34735de407743c3bb466ee2fcf3c332, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/65b9ee4dc1c04b47a626449e7afc338f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/6d607dbffeae4df999d6c4752a4cb5d4, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/1fff185962d6466cbd13022de644e41d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a1764e86b7864a2b98509821609ba3ad, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/5465fa9111a244f682af34075f557dd7, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/770e0687cddf4c7891283f7f8add3feb, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/8348ef5e68474188b333eb3b6117757a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/948395e63bcc4e5c9b026189804918d0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/44379dc170ff4f078f3f99525935cc99, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/95c82e6c194f4b70828766a623f8b0b0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ee3ae67a129f4436a4b44c3195cce511, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/c2f390182e6a4f0e91eccda733be8572, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/15217424ec7a417ca6a096cb7a1f1b9b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7904e9afc72b4cba9ddd9b1b0f607975, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/3d1a92a9e6aa45258c69e01cc62c5114] to archive 2024-11-13T19:25:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:25:40,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/18b86c323e5442d0af0beee8108dc22d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/18b86c323e5442d0af0beee8108dc22d 2024-11-13T19:25:40,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/656aaec0ef8b4d679d118b2d0a21d53a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/656aaec0ef8b4d679d118b2d0a21d53a 2024-11-13T19:25:40,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7fcf72c77d9e469487cd645a14d537af to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7fcf72c77d9e469487cd645a14d537af 2024-11-13T19:25:40,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/dcb3f5306bd14e969b1ed35d25e95ca2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/dcb3f5306bd14e969b1ed35d25e95ca2 2024-11-13T19:25:40,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/cc0a931ac70642b991ddf6dc3746e637 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/cc0a931ac70642b991ddf6dc3746e637 2024-11-13T19:25:40,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ce88e46ad93e49769b64fb6e2aa99645 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ce88e46ad93e49769b64fb6e2aa99645 2024-11-13T19:25:40,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a34735de407743c3bb466ee2fcf3c332 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a34735de407743c3bb466ee2fcf3c332 2024-11-13T19:25:40,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/65b9ee4dc1c04b47a626449e7afc338f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/65b9ee4dc1c04b47a626449e7afc338f 2024-11-13T19:25:40,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/6d607dbffeae4df999d6c4752a4cb5d4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/6d607dbffeae4df999d6c4752a4cb5d4 2024-11-13T19:25:40,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/1fff185962d6466cbd13022de644e41d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/1fff185962d6466cbd13022de644e41d 2024-11-13T19:25:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a1764e86b7864a2b98509821609ba3ad to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/a1764e86b7864a2b98509821609ba3ad 2024-11-13T19:25:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/5465fa9111a244f682af34075f557dd7 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/5465fa9111a244f682af34075f557dd7 2024-11-13T19:25:40,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/770e0687cddf4c7891283f7f8add3feb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/770e0687cddf4c7891283f7f8add3feb 2024-11-13T19:25:40,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/8348ef5e68474188b333eb3b6117757a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/8348ef5e68474188b333eb3b6117757a 2024-11-13T19:25:40,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/948395e63bcc4e5c9b026189804918d0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/948395e63bcc4e5c9b026189804918d0 2024-11-13T19:25:40,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/44379dc170ff4f078f3f99525935cc99 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/44379dc170ff4f078f3f99525935cc99 2024-11-13T19:25:40,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/95c82e6c194f4b70828766a623f8b0b0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/95c82e6c194f4b70828766a623f8b0b0 2024-11-13T19:25:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ee3ae67a129f4436a4b44c3195cce511 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/ee3ae67a129f4436a4b44c3195cce511 2024-11-13T19:25:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/c2f390182e6a4f0e91eccda733be8572 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/c2f390182e6a4f0e91eccda733be8572 2024-11-13T19:25:40,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/15217424ec7a417ca6a096cb7a1f1b9b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/15217424ec7a417ca6a096cb7a1f1b9b 2024-11-13T19:25:40,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7904e9afc72b4cba9ddd9b1b0f607975 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/7904e9afc72b4cba9ddd9b1b0f607975 2024-11-13T19:25:40,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/3d1a92a9e6aa45258c69e01cc62c5114 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/3d1a92a9e6aa45258c69e01cc62c5114 2024-11-13T19:25:40,752 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:25:40,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/376afc0486dc4d66ab7e049969628ea5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1058a8ded8c54f07a186863c0370c474, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8a76c6737c3b40249394803a6e316300, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b6e9b3e8cf0c4260a77c564556003f66, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/0302e4d5c8074032ab94d92ffd64b532, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/7fc44b05268f478fb75f32a67bafff6d, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9a0f86e397514ba6a7e22c7075f22e74, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/cd32fc8c952f4eb9b62be4f180c055c2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a08f1b111f174deb892b27e579c8857e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f0f805e29b614364b93923b8170512d3, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/49fc633589c54b21b5a2b84fc82af3c1, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a5fa7a79d96d4462ad6509b2b05a4c85, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8f8c908598d943cfb153d57684276272, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9d949cdfc1d64eeb8697f87e5ef46c69, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/633bed984575485abdae823e7079c18f, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/13fd99a9cbb7449a8d69d64af872d595, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a56c1240f1764005aed0eed10e2add12, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1d7ae2f67d73415f87bc821ee72154dc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/dbcd3928ee70451f99b37dd490199059, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/20c8cf3c0c2d4697b58b856e8e05ffe9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/ac724dd220cd4b0fb7baa4e1da4a0929, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b389f4c44b5f43d68194aab0a2fed5e2] to archive 2024-11-13T19:25:40,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:25:40,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/376afc0486dc4d66ab7e049969628ea5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/376afc0486dc4d66ab7e049969628ea5 2024-11-13T19:25:40,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1058a8ded8c54f07a186863c0370c474 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1058a8ded8c54f07a186863c0370c474 2024-11-13T19:25:40,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8a76c6737c3b40249394803a6e316300 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8a76c6737c3b40249394803a6e316300 2024-11-13T19:25:40,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b6e9b3e8cf0c4260a77c564556003f66 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b6e9b3e8cf0c4260a77c564556003f66 2024-11-13T19:25:40,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/0302e4d5c8074032ab94d92ffd64b532 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/0302e4d5c8074032ab94d92ffd64b532 2024-11-13T19:25:40,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/7fc44b05268f478fb75f32a67bafff6d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/7fc44b05268f478fb75f32a67bafff6d 2024-11-13T19:25:40,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9a0f86e397514ba6a7e22c7075f22e74 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9a0f86e397514ba6a7e22c7075f22e74 2024-11-13T19:25:40,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/cd32fc8c952f4eb9b62be4f180c055c2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/cd32fc8c952f4eb9b62be4f180c055c2 2024-11-13T19:25:40,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a08f1b111f174deb892b27e579c8857e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a08f1b111f174deb892b27e579c8857e 2024-11-13T19:25:40,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f0f805e29b614364b93923b8170512d3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f0f805e29b614364b93923b8170512d3 2024-11-13T19:25:40,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/49fc633589c54b21b5a2b84fc82af3c1 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/49fc633589c54b21b5a2b84fc82af3c1 2024-11-13T19:25:40,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a5fa7a79d96d4462ad6509b2b05a4c85 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a5fa7a79d96d4462ad6509b2b05a4c85 2024-11-13T19:25:40,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8f8c908598d943cfb153d57684276272 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/8f8c908598d943cfb153d57684276272 2024-11-13T19:25:40,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9d949cdfc1d64eeb8697f87e5ef46c69 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/9d949cdfc1d64eeb8697f87e5ef46c69 2024-11-13T19:25:40,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/633bed984575485abdae823e7079c18f to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/633bed984575485abdae823e7079c18f 2024-11-13T19:25:40,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/13fd99a9cbb7449a8d69d64af872d595 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/13fd99a9cbb7449a8d69d64af872d595 2024-11-13T19:25:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a56c1240f1764005aed0eed10e2add12 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/a56c1240f1764005aed0eed10e2add12 2024-11-13T19:25:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1d7ae2f67d73415f87bc821ee72154dc to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/1d7ae2f67d73415f87bc821ee72154dc 2024-11-13T19:25:40,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/dbcd3928ee70451f99b37dd490199059 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/dbcd3928ee70451f99b37dd490199059 2024-11-13T19:25:40,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/20c8cf3c0c2d4697b58b856e8e05ffe9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/20c8cf3c0c2d4697b58b856e8e05ffe9 2024-11-13T19:25:40,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/ac724dd220cd4b0fb7baa4e1da4a0929 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/ac724dd220cd4b0fb7baa4e1da4a0929 2024-11-13T19:25:40,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b389f4c44b5f43d68194aab0a2fed5e2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/b389f4c44b5f43d68194aab0a2fed5e2 2024-11-13T19:25:40,771 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=1 2024-11-13T19:25:40,771 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. 2024-11-13T19:25:40,771 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] regionserver.HRegion(1676): Region close journal for d17e3cc0ba121595297bf675d9f1d2e3: Waiting for close lock at 1731525939014Running coprocessor pre-close hooks at 1731525939014Disabling compacts and flushes for region at 1731525939014Disabling writes for close at 1731525939859 (+845 ms)Obtaining lock to block concurrent updates at 1731525939859Preparing flush snapshotting stores in d17e3cc0ba121595297bf675d9f1d2e3 at 1731525939859Finished memstore snapshotting TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731525939860 (+1 ms)Flushing stores of TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3. at 1731525939860Flushing d17e3cc0ba121595297bf675d9f1d2e3/A: creating writer at 1731525939860Flushing d17e3cc0ba121595297bf675d9f1d2e3/A: appending metadata at 1731525939863 (+3 ms)Flushing d17e3cc0ba121595297bf675d9f1d2e3/A: closing flushed file at 1731525939863Flushing d17e3cc0ba121595297bf675d9f1d2e3/B: creating writer at 1731525939873 (+10 ms)Flushing d17e3cc0ba121595297bf675d9f1d2e3/B: appending metadata at 1731525939877 (+4 ms)Flushing d17e3cc0ba121595297bf675d9f1d2e3/B: closing flushed file at 1731525939877Flushing d17e3cc0ba121595297bf675d9f1d2e3/C: creating writer at 1731525940287 (+410 ms)Flushing d17e3cc0ba121595297bf675d9f1d2e3/C: appending metadata at 1731525940291 (+4 ms)Flushing d17e3cc0ba121595297bf675d9f1d2e3/C: closing flushed file at 1731525940291Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5da8d4cb: reopening flushed file at 1731525940707 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32cb325f: reopening flushed file at 1731525940710 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@657edf52: reopening flushed file at 1731525940714 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d17e3cc0ba121595297bf675d9f1d2e3 in 859ms, sequenceid=334, compaction requested=true at 1731525940718 (+4 ms)Writing region close event to WAL at 1731525940768 (+50 ms)Running coprocessor post-close hooks at 1731525940771 (+3 ms)Closed at 1731525940771 2024-11-13T19:25:40,773 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=136}] handler.UnassignRegionHandler(157): Closed d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:40,774 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=d17e3cc0ba121595297bf675d9f1d2e3, regionState=CLOSED 2024-11-13T19:25:40,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; CloseRegionProcedure d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:25:40,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-13T19:25:40,777 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; CloseRegionProcedure d17e3cc0ba121595297bf675d9f1d2e3, server=e4b56bc8c4b3,40371,1731525792389 in 1.9150 sec 2024-11-13T19:25:40,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-13T19:25:40,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d17e3cc0ba121595297bf675d9f1d2e3, UNASSIGN in 1.9190 sec 2024-11-13T19:25:40,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-13T19:25:40,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9230 sec 2024-11-13T19:25:40,781 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525940781"}]},"ts":"1731525940781"} 2024-11-13T19:25:40,782 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:25:40,782 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:25:40,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9340 sec 2024-11-13T19:25:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-13T19:25:41,013 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:41,015 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:25:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:41,020 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=137, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T19:25:41,022 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=137, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:41,024 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:41,027 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/recovered.edits] 2024-11-13T19:25:41,030 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/4ba51273ab434a9587b4d5913efa7e56 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/4ba51273ab434a9587b4d5913efa7e56 2024-11-13T19:25:41,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/655e48f5b04f4182bdb9a735fed6ddb1 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/655e48f5b04f4182bdb9a735fed6ddb1 2024-11-13T19:25:41,033 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/690a438b8d174a5b88cfcd9d2fe34a56 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/A/690a438b8d174a5b88cfcd9d2fe34a56 2024-11-13T19:25:41,035 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/05325bcf05b04104a2bb070d6b0bee4a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/05325bcf05b04104a2bb070d6b0bee4a 2024-11-13T19:25:41,036 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/344be036a7694e86ad8588d39da1c75c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/344be036a7694e86ad8588d39da1c75c 2024-11-13T19:25:41,038 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/aa87a9bc6b574a40ba461769cfd7036c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/B/aa87a9bc6b574a40ba461769cfd7036c 2024-11-13T19:25:41,040 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/650e0abdd33e4e2687111061b4bccb27 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/650e0abdd33e4e2687111061b4bccb27 2024-11-13T19:25:41,042 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/85a59f323f534b22881cb2c76487e583 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/85a59f323f534b22881cb2c76487e583 2024-11-13T19:25:41,043 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f3817affdc79485fa055a21f869a4957 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/C/f3817affdc79485fa055a21f869a4957 2024-11-13T19:25:41,046 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/recovered.edits/337.seqid to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3/recovered.edits/337.seqid 2024-11-13T19:25:41,047 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/d17e3cc0ba121595297bf675d9f1d2e3 2024-11-13T19:25:41,047 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:25:41,049 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=137, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:41,051 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:25:41,053 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:25:41,054 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=137, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:41,054 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:25:41,055 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731525941054"}]},"ts":"9223372036854775807"} 2024-11-13T19:25:41,057 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:25:41,057 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d17e3cc0ba121595297bf675d9f1d2e3, NAME => 'TestAcidGuarantees,,1731525915065.d17e3cc0ba121595297bf675d9f1d2e3.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:25:41,057 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:25:41,057 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731525941057"}]},"ts":"9223372036854775807"} 2024-11-13T19:25:41,059 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:25:41,060 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=137, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:41,062 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-13T19:25:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-13T19:25:41,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:25:41,139 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:41,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=240 (was 241), OpenFileDescriptor=447 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=689 (was 824), ProcessCount=11 (was 11), AvailableMemoryMB=2560 (was 2584) 2024-11-13T19:25:41,155 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=689, ProcessCount=11, AvailableMemoryMB=2560 2024-11-13T19:25:41,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:25:41,156 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:25:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:41,179 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T19:25:41,179 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:41,179 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 138 2024-11-13T19:25:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:25:41,181 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T19:25:41,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742310_1486 (size=959) 2024-11-13T19:25:41,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:25:41,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:25:41,594 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a 2024-11-13T19:25:41,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742311_1487 (size=53) 2024-11-13T19:25:41,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:25:41,985 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:25:42,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:25:42,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing df37f3e4637a53b1d3fb1a07332c3cdb, disabling compactions & flushes 2024-11-13T19:25:42,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:42,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:42,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. after waiting 0 ms 2024-11-13T19:25:42,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:42,004 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:42,004 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for df37f3e4637a53b1d3fb1a07332c3cdb: Waiting for close lock at 1731525942004Disabling compacts and flushes for region at 1731525942004Disabling writes for close at 1731525942004Writing region close event to WAL at 1731525942004Closed at 1731525942004 2024-11-13T19:25:42,005 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T19:25:42,005 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731525942005"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731525942005"}]},"ts":"1731525942005"} 2024-11-13T19:25:42,007 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T19:25:42,008 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T19:25:42,008 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525942008"}]},"ts":"1731525942008"} 2024-11-13T19:25:42,010 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-13T19:25:42,010 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, ASSIGN}] 2024-11-13T19:25:42,012 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, ASSIGN 2024-11-13T19:25:42,013 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, ASSIGN; state=OFFLINE, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=false 2024-11-13T19:25:42,164 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:42,168 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, ASSIGN because future has completed 2024-11-13T19:25:42,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:25:42,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:25:42,330 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:42,331 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7752): Opening region: {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:25:42,331 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,332 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:25:42,332 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7794): checking encryption for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,332 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(7797): checking classloading for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,334 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,335 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37f3e4637a53b1d3fb1a07332c3cdb columnFamilyName A 2024-11-13T19:25:42,335 DEBUG [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:42,336 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(327): Store=df37f3e4637a53b1d3fb1a07332c3cdb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:42,336 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,337 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37f3e4637a53b1d3fb1a07332c3cdb columnFamilyName B 2024-11-13T19:25:42,337 DEBUG [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:42,338 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(327): Store=df37f3e4637a53b1d3fb1a07332c3cdb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:42,338 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,339 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37f3e4637a53b1d3fb1a07332c3cdb columnFamilyName C 2024-11-13T19:25:42,339 DEBUG [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:42,340 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(327): Store=df37f3e4637a53b1d3fb1a07332c3cdb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:42,340 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1038): replaying wal for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,341 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,341 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,342 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1048): stopping wal replay for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,342 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1060): Cleaning up temporary data for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,343 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:25:42,344 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1093): writing seq id for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,346 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T19:25:42,346 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1114): Opened df37f3e4637a53b1d3fb1a07332c3cdb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68660665, jitterRate=0.023123636841773987}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:25:42,347 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:42,347 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegion(1006): Region open journal for df37f3e4637a53b1d3fb1a07332c3cdb: Running coprocessor pre-open hook at 1731525942332Writing region info on filesystem at 1731525942332Initializing all the Stores at 1731525942333 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525942333Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525942333Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525942334 (+1 ms)Cleaning up temporary data from old regions at 1731525942342 (+8 ms)Running coprocessor post-open hooks at 1731525942347 (+5 ms)Region opened successfully at 1731525942347 2024-11-13T19:25:42,348 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., pid=140, masterSystemTime=1731525942323 2024-11-13T19:25:42,350 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:42,350 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=140}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:42,350 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=OPEN, openSeqNum=2, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:42,352 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:25:42,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-13T19:25:42,354 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 in 184 msec 2024-11-13T19:25:42,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-13T19:25:42,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, ASSIGN in 344 msec 2024-11-13T19:25:42,357 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T19:25:42,357 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525942357"}]},"ts":"1731525942357"} 2024-11-13T19:25:42,358 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-13T19:25:42,359 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=138, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T19:25:42,362 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2030 sec 2024-11-13T19:25:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-13T19:25:43,343 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:43,346 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:43,346 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:43,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:43,349 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fb509a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:43,349 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:43,350 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:43,351 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5665260e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:43,352 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:43,353 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:43,354 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:43,354 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T19:25:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:25:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4ff338f8 2024-11-13T19:25:43,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T19:25:43,356 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T19:25:43,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-13T19:25:43,358 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T19:25:43,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-13T19:25:43,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742312_1488 (size=995) 2024-11-13T19:25:43,772 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-13T19:25:43,772 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-13T19:25:43,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:25:43,781 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, REOPEN/MOVE}] 2024-11-13T19:25:43,782 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, REOPEN/MOVE 2024-11-13T19:25:43,784 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:43,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, REOPEN/MOVE because future has completed 2024-11-13T19:25:43,786 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:25:43,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:25:43,940 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:43,941 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:25:43,941 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing df37f3e4637a53b1d3fb1a07332c3cdb, disabling compactions & flushes 2024-11-13T19:25:43,941 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:43,941 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:43,941 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. after waiting 0 ms 2024-11-13T19:25:43,941 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:43,948 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T19:25:43,948 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:43,949 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for df37f3e4637a53b1d3fb1a07332c3cdb: Waiting for close lock at 1731525943941Running coprocessor pre-close hooks at 1731525943941Disabling compacts and flushes for region at 1731525943941Disabling writes for close at 1731525943941Writing region close event to WAL at 1731525943942 (+1 ms)Running coprocessor post-close hooks at 1731525943948 (+6 ms)Closed at 1731525943948 2024-11-13T19:25:43,949 WARN [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionServer(3299): Not adding moved region record: df37f3e4637a53b1d3fb1a07332c3cdb to self. 2024-11-13T19:25:43,951 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:43,951 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=CLOSED 2024-11-13T19:25:43,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:25:43,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-13T19:25:43,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 in 168 msec 2024-11-13T19:25:43,957 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, REOPEN/MOVE; state=CLOSED, location=e4b56bc8c4b3,40371,1731525792389; forceNewPlan=false, retain=true 2024-11-13T19:25:44,108 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=OPENING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, REOPEN/MOVE because future has completed 2024-11-13T19:25:44,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:25:44,272 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,273 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7752): Opening region: {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} 2024-11-13T19:25:44,273 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,273 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T19:25:44,274 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7794): checking encryption for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,274 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(7797): checking classloading for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,276 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,277 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37f3e4637a53b1d3fb1a07332c3cdb columnFamilyName A 2024-11-13T19:25:44,279 DEBUG [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:44,279 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(327): Store=df37f3e4637a53b1d3fb1a07332c3cdb/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:44,280 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,280 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37f3e4637a53b1d3fb1a07332c3cdb columnFamilyName B 2024-11-13T19:25:44,280 DEBUG [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:44,281 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(327): Store=df37f3e4637a53b1d3fb1a07332c3cdb/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:44,281 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,282 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region df37f3e4637a53b1d3fb1a07332c3cdb columnFamilyName C 2024-11-13T19:25:44,282 DEBUG [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:44,282 INFO [StoreOpener-df37f3e4637a53b1d3fb1a07332c3cdb-1 {}] regionserver.HStore(327): Store=df37f3e4637a53b1d3fb1a07332c3cdb/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T19:25:44,282 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1038): replaying wal for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,283 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,284 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,285 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1048): stopping wal replay for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,285 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1060): Cleaning up temporary data for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,286 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T19:25:44,287 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1093): writing seq id for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,287 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1114): Opened df37f3e4637a53b1d3fb1a07332c3cdb; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70017370, jitterRate=0.04334011673927307}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T19:25:44,288 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1122): Running coprocessor post-open hooks for df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,288 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegion(1006): Region open journal for df37f3e4637a53b1d3fb1a07332c3cdb: Running coprocessor pre-open hook at 1731525944274Writing region info on filesystem at 1731525944274Initializing all the Stores at 1731525944275 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525944276 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525944276Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731525944276Cleaning up temporary data from old regions at 1731525944285 (+9 ms)Running coprocessor post-open hooks at 1731525944288 (+3 ms)Region opened successfully at 1731525944288 2024-11-13T19:25:44,289 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., pid=145, masterSystemTime=1731525944266 2024-11-13T19:25:44,290 DEBUG [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,290 INFO [RS_OPEN_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_OPEN_REGION, pid=145}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,291 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=OPEN, openSeqNum=5, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,292 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:25:44,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=143 2024-11-13T19:25:44,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 in 181 msec 2024-11-13T19:25:44,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-13T19:25:44,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, REOPEN/MOVE in 514 msec 2024-11-13T19:25:44,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-13T19:25:44,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-13T19:25:44,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-13T19:25:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-13T19:25:44,300 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:44,301 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,301 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,301 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,302 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379fc09c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,303 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d609611, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,304 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d786e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,305 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca9309, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,306 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,307 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4692821d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,308 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da0380b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,309 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,309 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,309 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,309 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2faf298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,310 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,310 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,311 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43e48bae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,311 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,311 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,311 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,312 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229f9243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,313 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65077ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,314 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d7b1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,315 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7b9f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,317 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,317 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,317 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dedfe2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,318 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,319 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5304ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,321 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@612f00be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,322 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,323 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34f4f12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,324 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,324 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,324 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,324 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df1a583, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,326 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c2d37a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e4b56bc8c4b3,44929,-1 for getting cluster id 2024-11-13T19:25:44,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T19:25:44,328 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5' 2024-11-13T19:25:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T19:25:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4b4ec6e7-ba93-44d6-a3fa-71fa7dad5ed5" 2024-11-13T19:25:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@546cd54a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e4b56bc8c4b3,44929,-1] 2024-11-13T19:25:44,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T19:25:44,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:25:44,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T19:25:44,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111de12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T19:25:44,330 DEBUG [Thread-2210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,330 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,330 DEBUG [Thread-2203 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,331 DEBUG [Thread-2197 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,331 DEBUG [Thread-2201 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,331 DEBUG [Thread-2205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,331 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,332 DEBUG [Thread-2199 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,332 DEBUG [Thread-2216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,333 DEBUG [Thread-2208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45834, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,334 DEBUG [Thread-2214 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T19:25:44,334 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,334 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45844, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-13T19:25:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45864, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=-1] 2024-11-13T19:25:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T19:25:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:25:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,337 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:44,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,338 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:44,338 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,338 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T19:25:44,341 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5] 2024-11-13T19:25:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:44,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-13T19:25:44,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526004371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526004371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526004372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526004373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526004373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111347759aae28044be49bdfce01eb114fdc_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525944342/Put/seqid=0 2024-11-13T19:25:44,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742313_1489 (size=12154) 2024-11-13T19:25:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:25:44,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526004481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526004481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526004482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526004482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526004482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:25:44,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526004691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526004692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526004692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526004692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526004692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:44,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:44,780 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:44,783 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111347759aae28044be49bdfce01eb114fdc_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347759aae28044be49bdfce01eb114fdc_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:44,783 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/b6d1ade74993467a8fc125557c930c91, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:44,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/b6d1ade74993467a8fc125557c930c91 is 175, key is test_row_0/A:col10/1731525944342/Put/seqid=0 2024-11-13T19:25:44,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742314_1490 (size=30955) 2024-11-13T19:25:44,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:44,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:44,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:25:45,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526005001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526005002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526005002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526005002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526005002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,187 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=19, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/b6d1ade74993467a8fc125557c930c91 2024-11-13T19:25:45,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/7653bb3c7636422f81f36bb76dcafeed is 50, key is test_row_0/B:col10/1731525944342/Put/seqid=0 2024-11-13T19:25:45,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742315_1491 (size=12001) 2024-11-13T19:25:45,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:45,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:25:45,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526005512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526005512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526005512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526005512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526005512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:45,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:45,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/7653bb3c7636422f81f36bb76dcafeed 2024-11-13T19:25:45,636 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/fefec55da237452b9ff12bf2359d6414 is 50, key is test_row_0/C:col10/1731525944342/Put/seqid=0 2024-11-13T19:25:45,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742316_1492 (size=12001) 2024-11-13T19:25:45,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:45,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,711 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:45,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:46,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:46,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:46,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:46,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:46,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=19 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/fefec55da237452b9ff12bf2359d6414 2024-11-13T19:25:46,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/b6d1ade74993467a8fc125557c930c91 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/b6d1ade74993467a8fc125557c930c91 2024-11-13T19:25:46,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/b6d1ade74993467a8fc125557c930c91, entries=150, sequenceid=19, filesize=30.2 K 2024-11-13T19:25:46,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/7653bb3c7636422f81f36bb76dcafeed as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7653bb3c7636422f81f36bb76dcafeed 2024-11-13T19:25:46,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7653bb3c7636422f81f36bb76dcafeed, entries=150, sequenceid=19, filesize=11.7 K 2024-11-13T19:25:46,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/fefec55da237452b9ff12bf2359d6414 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/fefec55da237452b9ff12bf2359d6414 2024-11-13T19:25:46,051 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/fefec55da237452b9ff12bf2359d6414, entries=150, sequenceid=19, filesize=11.7 K 2024-11-13T19:25:46,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for df37f3e4637a53b1d3fb1a07332c3cdb in 1705ms, sequenceid=19, compaction requested=false 2024-11-13T19:25:46,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:46,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-13T19:25:46,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:46,169 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T19:25:46,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113acabf865c55640b2b6b2cf7f625912de_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525944369/Put/seqid=0 2024-11-13T19:25:46,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742317_1493 (size=12154) 2024-11-13T19:25:46,289 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T19:25:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:25:46,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:46,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526006532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526006535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,538 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526006535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526006536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526006536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:46,631 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113acabf865c55640b2b6b2cf7f625912de_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113acabf865c55640b2b6b2cf7f625912de_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:46,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/03f418409b194ac48fe7183e5e36db0c, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:46,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/03f418409b194ac48fe7183e5e36db0c is 175, key is test_row_0/A:col10/1731525944369/Put/seqid=0 2024-11-13T19:25:46,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742318_1494 (size=30955) 2024-11-13T19:25:46,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526006642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526006642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526006642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526006642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526006651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526006852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526006852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526006852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526006852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,855 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:46,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526006861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,035 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/03f418409b194ac48fe7183e5e36db0c 2024-11-13T19:25:47,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/1cb656695e2e4bcf9c88903a4361a816 is 50, key is test_row_0/B:col10/1731525944369/Put/seqid=0 2024-11-13T19:25:47,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742319_1495 (size=12001) 2024-11-13T19:25:47,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526007161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526007162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,164 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526007162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526007162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526007171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,444 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/1cb656695e2e4bcf9c88903a4361a816 2024-11-13T19:25:47,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/1269e157f1684be1b8f9172f3841b5a5 is 50, key is test_row_0/C:col10/1731525944369/Put/seqid=0 2024-11-13T19:25:47,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742320_1496 (size=12001) 2024-11-13T19:25:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526007671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,673 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526007671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526007671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526007671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:47,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526007681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:47,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:47,856 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/1269e157f1684be1b8f9172f3841b5a5 2024-11-13T19:25:47,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/03f418409b194ac48fe7183e5e36db0c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/03f418409b194ac48fe7183e5e36db0c 2024-11-13T19:25:47,863 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/03f418409b194ac48fe7183e5e36db0c, entries=150, sequenceid=40, filesize=30.2 K 2024-11-13T19:25:47,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/1cb656695e2e4bcf9c88903a4361a816 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/1cb656695e2e4bcf9c88903a4361a816 2024-11-13T19:25:47,866 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/1cb656695e2e4bcf9c88903a4361a816, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T19:25:47,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/1269e157f1684be1b8f9172f3841b5a5 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1269e157f1684be1b8f9172f3841b5a5 2024-11-13T19:25:47,870 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1269e157f1684be1b8f9172f3841b5a5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-13T19:25:47,870 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for df37f3e4637a53b1d3fb1a07332c3cdb in 1701ms, sequenceid=40, compaction requested=false 2024-11-13T19:25:47,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:47,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:47,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-13T19:25:47,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-13T19:25:47,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-13T19:25:47,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5340 sec 2024-11-13T19:25:47,875 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 3.5430 sec 2024-11-13T19:25:48,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-13T19:25:48,512 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:48,512 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-13T19:25:48,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:25:48,513 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:48,514 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:48,514 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:25:48,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-13T19:25:48,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:48,666 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T19:25:48,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379ef4dd9e17843a3aefdc1a9cb2b0032_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525946533/Put/seqid=0 2024-11-13T19:25:48,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742321_1497 (size=12154) 2024-11-13T19:25:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:48,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:48,690 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111379ef4dd9e17843a3aefdc1a9cb2b0032_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379ef4dd9e17843a3aefdc1a9cb2b0032_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:48,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/a4cd4ce9d9174d3bb4db40965299efc0, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:48,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/a4cd4ce9d9174d3bb4db40965299efc0 is 175, key is test_row_0/A:col10/1731525946533/Put/seqid=0 2024-11-13T19:25:48,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742322_1498 (size=30955) 2024-11-13T19:25:48,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526008701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,707 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526008705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526008706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526008706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526008707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526008811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526008822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526008822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526008822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526008822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:25:49,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526009021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526009032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526009032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526009032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526009032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,035 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,100 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/a4cd4ce9d9174d3bb4db40965299efc0 2024-11-13T19:25:49,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/e857a81bf02b41baa39105afcf28d920 is 50, key is test_row_0/B:col10/1731525946533/Put/seqid=0 2024-11-13T19:25:49,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742323_1499 (size=12001) 2024-11-13T19:25:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:25:49,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526009331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526009341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526009342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526009342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526009342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,346 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,513 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/e857a81bf02b41baa39105afcf28d920 2024-11-13T19:25:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/1cd384ee5b14445abf98165891e94fa9 is 50, key is test_row_0/C:col10/1731525946533/Put/seqid=0 2024-11-13T19:25:49,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742324_1500 (size=12001) 2024-11-13T19:25:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:25:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526009842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526009851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,853 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526009851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526009851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526009851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:49,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:49,927 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/1cd384ee5b14445abf98165891e94fa9 2024-11-13T19:25:49,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/a4cd4ce9d9174d3bb4db40965299efc0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a4cd4ce9d9174d3bb4db40965299efc0 2024-11-13T19:25:49,933 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a4cd4ce9d9174d3bb4db40965299efc0, entries=150, sequenceid=56, filesize=30.2 K 2024-11-13T19:25:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/e857a81bf02b41baa39105afcf28d920 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e857a81bf02b41baa39105afcf28d920 2024-11-13T19:25:49,936 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e857a81bf02b41baa39105afcf28d920, entries=150, sequenceid=56, filesize=11.7 K 2024-11-13T19:25:49,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/1cd384ee5b14445abf98165891e94fa9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1cd384ee5b14445abf98165891e94fa9 2024-11-13T19:25:49,939 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1cd384ee5b14445abf98165891e94fa9, entries=150, sequenceid=56, filesize=11.7 K 2024-11-13T19:25:49,940 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for df37f3e4637a53b1d3fb1a07332c3cdb in 1273ms, sequenceid=56, compaction requested=true 2024-11-13T19:25:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-13T19:25:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-13T19:25:49,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-13T19:25:49,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4270 sec 2024-11-13T19:25:49,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.4310 sec 2024-11-13T19:25:50,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-13T19:25:50,661 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:50,662 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:50,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-13T19:25:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:25:50,663 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:50,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:50,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:50,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:25:50,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-13T19:25:50,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:50,816 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T19:25:50,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111369803d4675314431a54939ccb6e60fbd_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525948704/Put/seqid=0 2024-11-13T19:25:50,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742325_1501 (size=12154) 2024-11-13T19:25:50,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:50,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:50,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526010877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526010877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526010878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526010878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526010879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:25:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526010991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526010991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526010991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526010992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,994 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:50,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:50,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526010992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:50,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:50,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526011201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526011201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526011202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526011202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526011202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,206 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:51,279 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111369803d4675314431a54939ccb6e60fbd_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111369803d4675314431a54939ccb6e60fbd_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:51,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bc118bdb7f224f46ac5063a75fcf5ca9, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:51,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bc118bdb7f224f46ac5063a75fcf5ca9 is 175, key is test_row_0/A:col10/1731525948704/Put/seqid=0 2024-11-13T19:25:51,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742326_1502 (size=30955) 2024-11-13T19:25:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:25:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526011511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526011511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,514 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526011512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526011512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526011512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:51,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:51,683 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bc118bdb7f224f46ac5063a75fcf5ca9 2024-11-13T19:25:51,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/6c03a3c1f6d640c0bfe7ba77cedc79dc is 50, key is test_row_0/B:col10/1731525948704/Put/seqid=0 2024-11-13T19:25:51,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742327_1503 (size=12001) 2024-11-13T19:25:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:25:51,985 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:25:51,985 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-13T19:25:52,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:52,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526012022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526012022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526012022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526012022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:52,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:52,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526012022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:52,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:52,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:52,092 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/6c03a3c1f6d640c0bfe7ba77cedc79dc 2024-11-13T19:25:52,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/47035347b6f34cee8775a499fc935804 is 50, key is test_row_0/C:col10/1731525948704/Put/seqid=0 2024-11-13T19:25:52,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742328_1504 (size=12001) 2024-11-13T19:25:52,502 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/47035347b6f34cee8775a499fc935804 2024-11-13T19:25:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bc118bdb7f224f46ac5063a75fcf5ca9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bc118bdb7f224f46ac5063a75fcf5ca9 2024-11-13T19:25:52,509 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bc118bdb7f224f46ac5063a75fcf5ca9, entries=150, sequenceid=77, filesize=30.2 K 2024-11-13T19:25:52,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/6c03a3c1f6d640c0bfe7ba77cedc79dc as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/6c03a3c1f6d640c0bfe7ba77cedc79dc 2024-11-13T19:25:52,513 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/6c03a3c1f6d640c0bfe7ba77cedc79dc, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T19:25:52,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/47035347b6f34cee8775a499fc935804 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/47035347b6f34cee8775a499fc935804 2024-11-13T19:25:52,516 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/47035347b6f34cee8775a499fc935804, entries=150, sequenceid=77, filesize=11.7 K 2024-11-13T19:25:52,517 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for df37f3e4637a53b1d3fb1a07332c3cdb in 1700ms, sequenceid=77, compaction requested=true 2024-11-13T19:25:52,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:52,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:52,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-13T19:25:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-13T19:25:52,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-13T19:25:52,520 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8540 sec 2024-11-13T19:25:52,521 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8580 sec 2024-11-13T19:25:52,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-13T19:25:52,812 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:52,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:52,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-13T19:25:52,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:25:52,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:52,814 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:52,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:52,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:25:52,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-13T19:25:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:52,967 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-13T19:25:52,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113880ff0fa6a6d44fd81427dcedb8ae32d_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525950878/Put/seqid=0 2024-11-13T19:25:52,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742329_1505 (size=12154) 2024-11-13T19:25:53,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:53,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:53,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526013084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526013084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526013085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526013085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,090 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526013090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:25:53,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526013191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526013202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526013202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526013202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,203 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526013202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:53,379 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113880ff0fa6a6d44fd81427dcedb8ae32d_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113880ff0fa6a6d44fd81427dcedb8ae32d_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/df55457959ce42028bbe6bfff478ad97, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:53,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/df55457959ce42028bbe6bfff478ad97 is 175, key is test_row_0/A:col10/1731525950878/Put/seqid=0 2024-11-13T19:25:53,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742330_1506 (size=30955) 2024-11-13T19:25:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526013401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526013412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526013412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526013412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526013412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,414 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:25:53,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526013712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526013722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526013722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,725 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526013722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526013722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:53,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:53,794 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/df55457959ce42028bbe6bfff478ad97 2024-11-13T19:25:53,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/de4d863c96804019852163a78373d42b is 50, key is test_row_0/B:col10/1731525950878/Put/seqid=0 2024-11-13T19:25:53,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742331_1507 (size=12001) 2024-11-13T19:25:53,807 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/de4d863c96804019852163a78373d42b 2024-11-13T19:25:53,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c3101091a3b846d1a91228958b3dc486 is 50, key is test_row_0/C:col10/1731525950878/Put/seqid=0 2024-11-13T19:25:53,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742332_1508 (size=12001) 2024-11-13T19:25:53,815 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c3101091a3b846d1a91228958b3dc486 2024-11-13T19:25:53,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/df55457959ce42028bbe6bfff478ad97 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/df55457959ce42028bbe6bfff478ad97 2024-11-13T19:25:53,822 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/df55457959ce42028bbe6bfff478ad97, entries=150, sequenceid=92, filesize=30.2 K 2024-11-13T19:25:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/de4d863c96804019852163a78373d42b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de4d863c96804019852163a78373d42b 2024-11-13T19:25:53,826 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de4d863c96804019852163a78373d42b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-13T19:25:53,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c3101091a3b846d1a91228958b3dc486 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c3101091a3b846d1a91228958b3dc486 2024-11-13T19:25:53,829 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c3101091a3b846d1a91228958b3dc486, entries=150, sequenceid=92, filesize=11.7 K 2024-11-13T19:25:53,830 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for df37f3e4637a53b1d3fb1a07332c3cdb in 864ms, sequenceid=92, compaction requested=true 2024-11-13T19:25:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:53,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-13T19:25:53,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-13T19:25:53,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-13T19:25:53,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-13T19:25:53,834 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.0220 sec 2024-11-13T19:25:53,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-13T19:25:53,951 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:53,951 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:53,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-13T19:25:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:25:53,953 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:53,953 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:53,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:54,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:25:54,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-13T19:25:54,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:54,105 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T19:25:54,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f74fa18317784821a192bf96993d82d1_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525953053/Put/seqid=0 2024-11-13T19:25:54,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742333_1509 (size=12154) 2024-11-13T19:25:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:54,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:54,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526014238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526014242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526014242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526014243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526014243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:25:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526014352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526014352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526014352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526014352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526014352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:54,517 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113f74fa18317784821a192bf96993d82d1_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f74fa18317784821a192bf96993d82d1_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:54,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/30136a48faf547c590a8b61b5160131e, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:54,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/30136a48faf547c590a8b61b5160131e is 175, key is test_row_0/A:col10/1731525953053/Put/seqid=0 2024-11-13T19:25:54,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742334_1510 (size=30955) 2024-11-13T19:25:54,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526014562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526014562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526014562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526014562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526014562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,565 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:25:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526014871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526014871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526014872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526014872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526014872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:54,921 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=113, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/30136a48faf547c590a8b61b5160131e 2024-11-13T19:25:54,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/de1a31602f4945d5bf220e2e5bdc7172 is 50, key is test_row_0/B:col10/1731525953053/Put/seqid=0 2024-11-13T19:25:54,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742335_1511 (size=12001) 2024-11-13T19:25:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:25:55,332 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/de1a31602f4945d5bf220e2e5bdc7172 2024-11-13T19:25:55,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/79111a1f19fb4784903c2e1a2f2411d8 is 50, key is test_row_0/C:col10/1731525953053/Put/seqid=0 2024-11-13T19:25:55,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742336_1512 (size=12001) 2024-11-13T19:25:55,348 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/79111a1f19fb4784903c2e1a2f2411d8 2024-11-13T19:25:55,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/30136a48faf547c590a8b61b5160131e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/30136a48faf547c590a8b61b5160131e 2024-11-13T19:25:55,354 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/30136a48faf547c590a8b61b5160131e, entries=150, sequenceid=113, filesize=30.2 K 2024-11-13T19:25:55,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/de1a31602f4945d5bf220e2e5bdc7172 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de1a31602f4945d5bf220e2e5bdc7172 2024-11-13T19:25:55,357 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de1a31602f4945d5bf220e2e5bdc7172, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T19:25:55,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/79111a1f19fb4784903c2e1a2f2411d8 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/79111a1f19fb4784903c2e1a2f2411d8 2024-11-13T19:25:55,360 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/79111a1f19fb4784903c2e1a2f2411d8, entries=150, sequenceid=113, filesize=11.7 K 2024-11-13T19:25:55,361 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for df37f3e4637a53b1d3fb1a07332c3cdb in 1256ms, sequenceid=113, compaction requested=true 2024-11-13T19:25:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:55,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-13T19:25:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-13T19:25:55,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-13T19:25:55,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-13T19:25:55,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4130 sec 2024-11-13T19:25:55,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:55,385 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:55,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-13T19:25:55,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113424a60f806f84c06824e17ba5c5f94ad_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525955384/Put/seqid=0 2024-11-13T19:25:55,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742337_1513 (size=14644) 2024-11-13T19:25:55,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526015407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,412 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526015410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526015410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526015412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526015412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,417 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526015521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526015521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,525 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526015521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526015521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526015522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526015731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526015731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526015731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526015731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:55,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526015731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:55,796 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:55,799 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113424a60f806f84c06824e17ba5c5f94ad_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113424a60f806f84c06824e17ba5c5f94ad_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:55,800 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/95bf705783294cd8a71510031820042c, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:55,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/95bf705783294cd8a71510031820042c is 175, key is test_row_0/A:col10/1731525955384/Put/seqid=0 2024-11-13T19:25:55,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742338_1514 (size=39599) 2024-11-13T19:25:56,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526016041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526016041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526016041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526016041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526016041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-13T19:25:56,102 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:56,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:56,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-13T19:25:56,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:25:56,104 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:56,105 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:56,105 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:56,210 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/95bf705783294cd8a71510031820042c 2024-11-13T19:25:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:25:56,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/d5266b5755474f87b398a3c9725f910c is 50, key is test_row_0/B:col10/1731525955384/Put/seqid=0 2024-11-13T19:25:56,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742339_1515 (size=12051) 2024-11-13T19:25:56,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:25:56,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:56,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:25:56,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:56,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:25:56,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526016551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,556 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526016551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526016551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526016552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526016552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:56,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:56,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:25:56,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:56,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/d5266b5755474f87b398a3c9725f910c 2024-11-13T19:25:56,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/aa6e9b50036440fca3a643c16acf3831 is 50, key is test_row_0/C:col10/1731525955384/Put/seqid=0 2024-11-13T19:25:56,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742340_1516 (size=12051) 2024-11-13T19:25:56,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:25:56,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:56,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:25:56,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:25:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:56,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:57,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:25:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:57,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:57,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:57,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/aa6e9b50036440fca3a643c16acf3831 2024-11-13T19:25:57,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/95bf705783294cd8a71510031820042c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/95bf705783294cd8a71510031820042c 2024-11-13T19:25:57,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/95bf705783294cd8a71510031820042c, entries=200, sequenceid=129, filesize=38.7 K 2024-11-13T19:25:57,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/d5266b5755474f87b398a3c9725f910c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d5266b5755474f87b398a3c9725f910c 2024-11-13T19:25:57,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d5266b5755474f87b398a3c9725f910c, entries=150, sequenceid=129, filesize=11.8 K 2024-11-13T19:25:57,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/aa6e9b50036440fca3a643c16acf3831 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/aa6e9b50036440fca3a643c16acf3831 2024-11-13T19:25:57,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/aa6e9b50036440fca3a643c16acf3831, entries=150, sequenceid=129, filesize=11.8 K 2024-11-13T19:25:57,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for df37f3e4637a53b1d3fb1a07332c3cdb in 1664ms, sequenceid=129, compaction requested=true 2024-11-13T19:25:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:57,050 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-13T19:25:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:57,050 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-13T19:25:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:57,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:57,051 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 225329 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-13T19:25:57,051 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84057 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-13T19:25:57,051 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/B is initiating minor compaction (all files) 2024-11-13T19:25:57,051 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/A is initiating minor compaction (all files) 2024-11-13T19:25:57,051 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/B in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:57,051 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/A in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:57,051 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7653bb3c7636422f81f36bb76dcafeed, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/1cb656695e2e4bcf9c88903a4361a816, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e857a81bf02b41baa39105afcf28d920, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/6c03a3c1f6d640c0bfe7ba77cedc79dc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de4d863c96804019852163a78373d42b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de1a31602f4945d5bf220e2e5bdc7172, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d5266b5755474f87b398a3c9725f910c] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=82.1 K 2024-11-13T19:25:57,051 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/b6d1ade74993467a8fc125557c930c91, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/03f418409b194ac48fe7183e5e36db0c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a4cd4ce9d9174d3bb4db40965299efc0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bc118bdb7f224f46ac5063a75fcf5ca9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/df55457959ce42028bbe6bfff478ad97, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/30136a48faf547c590a8b61b5160131e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/95bf705783294cd8a71510031820042c] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=220.0 K 2024-11-13T19:25:57,051 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:57,051 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/b6d1ade74993467a8fc125557c930c91, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/03f418409b194ac48fe7183e5e36db0c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a4cd4ce9d9174d3bb4db40965299efc0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bc118bdb7f224f46ac5063a75fcf5ca9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/df55457959ce42028bbe6bfff478ad97, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/30136a48faf547c590a8b61b5160131e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/95bf705783294cd8a71510031820042c] 2024-11-13T19:25:57,051 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7653bb3c7636422f81f36bb76dcafeed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731525944342 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6d1ade74993467a8fc125557c930c91, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731525944342 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb656695e2e4bcf9c88903a4361a816, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525944369 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03f418409b194ac48fe7183e5e36db0c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525944369 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e857a81bf02b41baa39105afcf28d920, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731525946533 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4cd4ce9d9174d3bb4db40965299efc0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731525946533 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c03a3c1f6d640c0bfe7ba77cedc79dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525948704 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc118bdb7f224f46ac5063a75fcf5ca9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525948704 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting de4d863c96804019852163a78373d42b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731525950877 2024-11-13T19:25:57,052 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting df55457959ce42028bbe6bfff478ad97, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731525950877 2024-11-13T19:25:57,053 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting de1a31602f4945d5bf220e2e5bdc7172, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731525953053 2024-11-13T19:25:57,053 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30136a48faf547c590a8b61b5160131e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731525953053 2024-11-13T19:25:57,053 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting d5266b5755474f87b398a3c9725f910c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525954236 2024-11-13T19:25:57,053 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95bf705783294cd8a71510031820042c, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525954234 2024-11-13T19:25:57,059 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:57,060 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113fec21ecfec9c44b2b90933afc4fe50d7_df37f3e4637a53b1d3fb1a07332c3cdb store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:57,063 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#B#compaction#439 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:57,063 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/3972469fc6324c6a99f2d537d21ed98a is 50, key is test_row_0/B:col10/1731525955384/Put/seqid=0 2024-11-13T19:25:57,064 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113fec21ecfec9c44b2b90933afc4fe50d7_df37f3e4637a53b1d3fb1a07332c3cdb, store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:57,065 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113fec21ecfec9c44b2b90933afc4fe50d7_df37f3e4637a53b1d3fb1a07332c3cdb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:57,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742341_1517 (size=12291) 2024-11-13T19:25:57,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742342_1518 (size=4469) 2024-11-13T19:25:57,068 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#A#compaction#438 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:57,068 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/a1407250136647b6beae3d55348c9080 is 175, key is test_row_0/A:col10/1731525955384/Put/seqid=0 2024-11-13T19:25:57,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742343_1519 (size=31245) 2024-11-13T19:25:57,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-13T19:25:57,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:57,184 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-13T19:25:57,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131a26732cb0154ab680d77f2de146b049_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525955411/Put/seqid=0 2024-11-13T19:25:57,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742344_1520 (size=12304) 2024-11-13T19:25:57,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:57,195 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411131a26732cb0154ab680d77f2de146b049_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a26732cb0154ab680d77f2de146b049_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:57,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/5ea399eee8de42adba2437cc2afeff24, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:57,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/5ea399eee8de42adba2437cc2afeff24 is 175, key is test_row_0/A:col10/1731525955411/Put/seqid=0 2024-11-13T19:25:57,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742345_1521 (size=31105) 2024-11-13T19:25:57,200 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/5ea399eee8de42adba2437cc2afeff24 2024-11-13T19:25:57,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/15409ca872c74ab9a0fadf9546c6a102 is 50, key is test_row_0/B:col10/1731525955411/Put/seqid=0 2024-11-13T19:25:57,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742346_1522 (size=12151) 2024-11-13T19:25:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:25:57,470 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/3972469fc6324c6a99f2d537d21ed98a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/3972469fc6324c6a99f2d537d21ed98a 2024-11-13T19:25:57,474 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/a1407250136647b6beae3d55348c9080 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a1407250136647b6beae3d55348c9080 2024-11-13T19:25:57,474 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/B of df37f3e4637a53b1d3fb1a07332c3cdb into 3972469fc6324c6a99f2d537d21ed98a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:57,474 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:57,474 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/B, priority=9, startTime=1731525957050; duration=0sec 2024-11-13T19:25:57,474 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:57,474 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:B 2024-11-13T19:25:57,474 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-13T19:25:57,476 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 84057 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-13T19:25:57,476 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/C is initiating minor compaction (all files) 2024-11-13T19:25:57,476 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/C in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:57,476 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/fefec55da237452b9ff12bf2359d6414, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1269e157f1684be1b8f9172f3841b5a5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1cd384ee5b14445abf98165891e94fa9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/47035347b6f34cee8775a499fc935804, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c3101091a3b846d1a91228958b3dc486, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/79111a1f19fb4784903c2e1a2f2411d8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/aa6e9b50036440fca3a643c16acf3831] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=82.1 K 2024-11-13T19:25:57,476 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting fefec55da237452b9ff12bf2359d6414, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=19, earliestPutTs=1731525944342 2024-11-13T19:25:57,476 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1269e157f1684be1b8f9172f3841b5a5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731525944369 2024-11-13T19:25:57,477 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cd384ee5b14445abf98165891e94fa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1731525946533 2024-11-13T19:25:57,478 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/A of df37f3e4637a53b1d3fb1a07332c3cdb into a1407250136647b6beae3d55348c9080(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:57,478 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:57,478 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/A, priority=9, startTime=1731525957050; duration=0sec 2024-11-13T19:25:57,478 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:57,478 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:A 2024-11-13T19:25:57,478 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 47035347b6f34cee8775a499fc935804, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731525948704 2024-11-13T19:25:57,478 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c3101091a3b846d1a91228958b3dc486, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731525950877 2024-11-13T19:25:57,478 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 79111a1f19fb4784903c2e1a2f2411d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731525953053 2024-11-13T19:25:57,479 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting aa6e9b50036440fca3a643c16acf3831, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525954236 2024-11-13T19:25:57,488 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#C#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:57,488 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/2a3bd4bb2c2641b591354ff1a360c153 is 50, key is test_row_0/C:col10/1731525955384/Put/seqid=0 2024-11-13T19:25:57,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742347_1523 (size=12291) 2024-11-13T19:25:57,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:57,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:57,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526017576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526017577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526017578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526017579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526017581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,609 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/15409ca872c74ab9a0fadf9546c6a102 2024-11-13T19:25:57,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c769eb07d96e43608ef8fe16b2953107 is 50, key is test_row_0/C:col10/1731525955411/Put/seqid=0 2024-11-13T19:25:57,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742348_1524 (size=12151) 2024-11-13T19:25:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526017692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526017692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526017692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526017692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526017692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,900 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/2a3bd4bb2c2641b591354ff1a360c153 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/2a3bd4bb2c2641b591354ff1a360c153 2024-11-13T19:25:57,903 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/C of df37f3e4637a53b1d3fb1a07332c3cdb into 2a3bd4bb2c2641b591354ff1a360c153(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:25:57,903 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:57,903 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/C, priority=9, startTime=1731525957050; duration=0sec 2024-11-13T19:25:57,903 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:57,903 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:C 2024-11-13T19:25:57,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526017902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526017902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526017902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526017902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:57,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526017902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,018 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c769eb07d96e43608ef8fe16b2953107 2024-11-13T19:25:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/5ea399eee8de42adba2437cc2afeff24 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/5ea399eee8de42adba2437cc2afeff24 2024-11-13T19:25:58,024 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/5ea399eee8de42adba2437cc2afeff24, entries=150, sequenceid=150, filesize=30.4 K 2024-11-13T19:25:58,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/15409ca872c74ab9a0fadf9546c6a102 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/15409ca872c74ab9a0fadf9546c6a102 2024-11-13T19:25:58,028 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/15409ca872c74ab9a0fadf9546c6a102, entries=150, sequenceid=150, filesize=11.9 K 2024-11-13T19:25:58,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c769eb07d96e43608ef8fe16b2953107 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c769eb07d96e43608ef8fe16b2953107 2024-11-13T19:25:58,031 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c769eb07d96e43608ef8fe16b2953107, entries=150, sequenceid=150, filesize=11.9 K 2024-11-13T19:25:58,032 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for df37f3e4637a53b1d3fb1a07332c3cdb in 848ms, sequenceid=150, compaction requested=false 2024-11-13T19:25:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-13T19:25:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-13T19:25:58,034 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-13T19:25:58,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9280 sec 2024-11-13T19:25:58,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.9330 sec 2024-11-13T19:25:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:58,214 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:25:58,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T19:25:58,221 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c09b196ac6d048179ae2c716596523e4_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525958213/Put/seqid=0 2024-11-13T19:25:58,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742349_1525 (size=14794) 2024-11-13T19:25:58,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526018232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526018233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526018234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526018234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,239 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526018238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,242 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-13T19:25:58,252 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:25:58,252 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:25:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-13T19:25:58,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:25:58,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:25:58,254 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:25:58,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:25:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526018341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526018341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526018342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526018342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526018351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:25:58,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:58,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:58,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,406 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526018551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526018552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526018552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526018552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:58,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:58,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,559 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,564 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526018561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:25:58,625 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:25:58,628 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113c09b196ac6d048179ae2c716596523e4_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c09b196ac6d048179ae2c716596523e4_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:25:58,628 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/fba8c70a66634b7db803fbbe194cadab, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:58,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/fba8c70a66634b7db803fbbe194cadab is 175, key is test_row_0/A:col10/1731525958213/Put/seqid=0 2024-11-13T19:25:58,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742350_1526 (size=39749) 2024-11-13T19:25:58,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:58,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526018861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526018861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:58,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526018862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526018862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,865 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526018871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:58,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:25:59,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/fba8c70a66634b7db803fbbe194cadab 2024-11-13T19:25:59,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/7b7e1fcef4e64e19b18998b081a3178b is 50, key is test_row_0/B:col10/1731525958213/Put/seqid=0 2024-11-13T19:25:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742351_1527 (size=12151) 2024-11-13T19:25:59,169 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:59,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526019372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:59,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:59,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526019372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:59,375 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:59,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526019372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526019372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,375 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:59,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:25:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526019381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:25:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:25:59,384 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:25:59,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:25:59,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/7b7e1fcef4e64e19b18998b081a3178b 2024-11-13T19:25:59,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/feff67efeaf64a558779a4b840e259d2 is 50, key is test_row_0/C:col10/1731525958213/Put/seqid=0 2024-11-13T19:25:59,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742352_1528 (size=12151) 2024-11-13T19:25:59,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,627 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:25:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:25:59,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/feff67efeaf64a558779a4b840e259d2 2024-11-13T19:25:59,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/fba8c70a66634b7db803fbbe194cadab as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/fba8c70a66634b7db803fbbe194cadab 2024-11-13T19:25:59,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/fba8c70a66634b7db803fbbe194cadab, entries=200, sequenceid=171, filesize=38.8 K 2024-11-13T19:25:59,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/7b7e1fcef4e64e19b18998b081a3178b as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7b7e1fcef4e64e19b18998b081a3178b 2024-11-13T19:25:59,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7b7e1fcef4e64e19b18998b081a3178b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T19:25:59,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/feff67efeaf64a558779a4b840e259d2 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/feff67efeaf64a558779a4b840e259d2 2024-11-13T19:25:59,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/feff67efeaf64a558779a4b840e259d2, entries=150, sequenceid=171, filesize=11.9 K 2024-11-13T19:25:59,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for df37f3e4637a53b1d3fb1a07332c3cdb in 1654ms, sequenceid=171, compaction requested=true 2024-11-13T19:25:59,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:25:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:25:59,869 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:25:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:25:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:25:59,869 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:25:59,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:25:59,869 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:59,869 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/A is initiating minor compaction (all files) 2024-11-13T19:25:59,869 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/A in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,869 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a1407250136647b6beae3d55348c9080, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/5ea399eee8de42adba2437cc2afeff24, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/fba8c70a66634b7db803fbbe194cadab] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=99.7 K 2024-11-13T19:25:59,869 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,870 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:25:59,870 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a1407250136647b6beae3d55348c9080, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/5ea399eee8de42adba2437cc2afeff24, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/fba8c70a66634b7db803fbbe194cadab] 2024-11-13T19:25:59,870 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/B is initiating minor compaction (all files) 2024-11-13T19:25:59,870 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/B in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,870 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/3972469fc6324c6a99f2d537d21ed98a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/15409ca872c74ab9a0fadf9546c6a102, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7b7e1fcef4e64e19b18998b081a3178b] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=35.7 K 2024-11-13T19:25:59,870 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 3972469fc6324c6a99f2d537d21ed98a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525954236 2024-11-13T19:25:59,870 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1407250136647b6beae3d55348c9080, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525954236 2024-11-13T19:25:59,870 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 15409ca872c74ab9a0fadf9546c6a102, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731525955408 2024-11-13T19:25:59,870 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ea399eee8de42adba2437cc2afeff24, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731525955408 2024-11-13T19:25:59,871 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b7e1fcef4e64e19b18998b081a3178b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525958213 2024-11-13T19:25:59,871 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting fba8c70a66634b7db803fbbe194cadab, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525957566 2024-11-13T19:25:59,875 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:59,877 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:59,877 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/e89e471618a943658712d2f175babb31 is 50, key is test_row_0/B:col10/1731525958213/Put/seqid=0 2024-11-13T19:25:59,881 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241113bb1e1e6e470f4c2aae1d324efc063ffb_df37f3e4637a53b1d3fb1a07332c3cdb store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:59,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742353_1529 (size=12493) 2024-11-13T19:25:59,883 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241113bb1e1e6e470f4c2aae1d324efc063ffb_df37f3e4637a53b1d3fb1a07332c3cdb, store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:59,884 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113bb1e1e6e470f4c2aae1d324efc063ffb_df37f3e4637a53b1d3fb1a07332c3cdb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:25:59,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742354_1530 (size=4469) 2024-11-13T19:25:59,888 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#A#compaction#448 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:25:59,889 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/3cf2427dbbab48d59b48bcbe82d307bf is 175, key is test_row_0/A:col10/1731525958213/Put/seqid=0 2024-11-13T19:25:59,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742355_1531 (size=31447) 2024-11-13T19:25:59,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-13T19:25:59,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:25:59,932 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T19:25:59,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113af2ad894c4c04ad3a725f2445fc9dad4_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525958230/Put/seqid=0 2024-11-13T19:25:59,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742356_1532 (size=12304) 2024-11-13T19:26:00,288 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/e89e471618a943658712d2f175babb31 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e89e471618a943658712d2f175babb31 2024-11-13T19:26:00,291 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/B of df37f3e4637a53b1d3fb1a07332c3cdb into e89e471618a943658712d2f175babb31(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:26:00,292 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:00,292 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/B, priority=13, startTime=1731525959869; duration=0sec 2024-11-13T19:26:00,292 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:26:00,292 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:B 2024-11-13T19:26:00,292 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:26:00,299 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:26:00,299 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/C is initiating minor compaction (all files) 2024-11-13T19:26:00,299 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/C in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:00,299 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/2a3bd4bb2c2641b591354ff1a360c153, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c769eb07d96e43608ef8fe16b2953107, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/feff67efeaf64a558779a4b840e259d2] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=35.7 K 2024-11-13T19:26:00,299 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a3bd4bb2c2641b591354ff1a360c153, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731525954236 2024-11-13T19:26:00,300 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c769eb07d96e43608ef8fe16b2953107, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731525955408 2024-11-13T19:26:00,300 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting feff67efeaf64a558779a4b840e259d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525958213 2024-11-13T19:26:00,302 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/3cf2427dbbab48d59b48bcbe82d307bf as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/3cf2427dbbab48d59b48bcbe82d307bf 2024-11-13T19:26:00,306 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:26:00,306 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/A of df37f3e4637a53b1d3fb1a07332c3cdb into 3cf2427dbbab48d59b48bcbe82d307bf(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:26:00,306 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:00,306 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/A, priority=13, startTime=1731525959868; duration=0sec 2024-11-13T19:26:00,306 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:26:00,306 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/20d4068469594bf595d4ca63c36982b0 is 50, key is test_row_0/C:col10/1731525958213/Put/seqid=0 2024-11-13T19:26:00,306 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:A 2024-11-13T19:26:00,324 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742357_1533 (size=12493) 2024-11-13T19:26:00,327 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/20d4068469594bf595d4ca63c36982b0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/20d4068469594bf595d4ca63c36982b0 2024-11-13T19:26:00,332 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/C of df37f3e4637a53b1d3fb1a07332c3cdb into 20d4068469594bf595d4ca63c36982b0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:26:00,332 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:00,332 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/C, priority=13, startTime=1731525959869; duration=0sec 2024-11-13T19:26:00,332 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:26:00,332 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:C 2024-11-13T19:26:00,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:26:00,344 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113af2ad894c4c04ad3a725f2445fc9dad4_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af2ad894c4c04ad3a725f2445fc9dad4_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:00,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bd5bcc90fcba4e48831378ec6d10a516, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:00,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bd5bcc90fcba4e48831378ec6d10a516 is 175, key is test_row_0/A:col10/1731525958230/Put/seqid=0 2024-11-13T19:26:00,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742358_1534 (size=31105) 2024-11-13T19:26:00,348 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bd5bcc90fcba4e48831378ec6d10a516 2024-11-13T19:26:00,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/c155d3a5027d4522bb0a238f1da3c668 is 50, key is test_row_0/B:col10/1731525958230/Put/seqid=0 2024-11-13T19:26:00,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742359_1535 (size=12151) 2024-11-13T19:26:00,360 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/c155d3a5027d4522bb0a238f1da3c668 2024-11-13T19:26:00,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/84ed3bf4ada64f9c99815a97a8328f57 is 50, key is test_row_0/C:col10/1731525958230/Put/seqid=0 2024-11-13T19:26:00,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742360_1536 (size=12151) 2024-11-13T19:26:00,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:00,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:00,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:26:00,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526020437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526020438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526020438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526020442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526020443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526020552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526020552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526020552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526020552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,555 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526020552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,555 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526020762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,765 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526020762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526020762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526020762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:00,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526020762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:00,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:00,773 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/84ed3bf4ada64f9c99815a97a8328f57 2024-11-13T19:26:00,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/bd5bcc90fcba4e48831378ec6d10a516 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bd5bcc90fcba4e48831378ec6d10a516 2024-11-13T19:26:00,779 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bd5bcc90fcba4e48831378ec6d10a516, entries=150, sequenceid=189, filesize=30.4 K 2024-11-13T19:26:00,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/c155d3a5027d4522bb0a238f1da3c668 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/c155d3a5027d4522bb0a238f1da3c668 2024-11-13T19:26:00,783 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/c155d3a5027d4522bb0a238f1da3c668, entries=150, sequenceid=189, filesize=11.9 K 2024-11-13T19:26:00,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/84ed3bf4ada64f9c99815a97a8328f57 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/84ed3bf4ada64f9c99815a97a8328f57 2024-11-13T19:26:00,786 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/84ed3bf4ada64f9c99815a97a8328f57, entries=150, sequenceid=189, filesize=11.9 K 2024-11-13T19:26:00,787 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for df37f3e4637a53b1d3fb1a07332c3cdb in 856ms, sequenceid=189, compaction requested=false 2024-11-13T19:26:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:00,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-13T19:26:00,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-13T19:26:00,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-13T19:26:00,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5340 sec 2024-11-13T19:26:00,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.5380 sec 2024-11-13T19:26:01,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:01,073 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:26:01,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-13T19:26:01,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111363a5c44534a74a38baa1e29c83089df0_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525961073/Put/seqid=0 2024-11-13T19:26:01,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526021085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526021086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526021086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526021087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742361_1537 (size=12304) 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,092 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:26:01,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526021088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,095 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111363a5c44534a74a38baa1e29c83089df0_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111363a5c44534a74a38baa1e29c83089df0_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:01,095 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/852d22cb75b84233824151b4979b33e3, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:01,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/852d22cb75b84233824151b4979b33e3 is 175, key is test_row_0/A:col10/1731525961073/Put/seqid=0 2024-11-13T19:26:01,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742362_1538 (size=31105) 2024-11-13T19:26:01,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526021201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526021201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526021202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526021202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526021202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526021411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526021411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,414 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526021411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526021412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,415 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526021412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,501 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/852d22cb75b84233824151b4979b33e3 2024-11-13T19:26:01,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/5160119bc374420ca0dc3f713d7d72cf is 50, key is test_row_0/B:col10/1731525961073/Put/seqid=0 2024-11-13T19:26:01,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742363_1539 (size=12151) 2024-11-13T19:26:01,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/5160119bc374420ca0dc3f713d7d72cf 2024-11-13T19:26:01,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c003d6d265024855bc44c9bbda2e7a98 is 50, key is test_row_0/C:col10/1731525961073/Put/seqid=0 2024-11-13T19:26:01,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742364_1540 (size=12151) 2024-11-13T19:26:01,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526021721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526021721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526021721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526021721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,724 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526021721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:01,725 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:01,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c003d6d265024855bc44c9bbda2e7a98 2024-11-13T19:26:01,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/852d22cb75b84233824151b4979b33e3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/852d22cb75b84233824151b4979b33e3 2024-11-13T19:26:01,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/852d22cb75b84233824151b4979b33e3, entries=150, sequenceid=212, filesize=30.4 K 2024-11-13T19:26:01,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/5160119bc374420ca0dc3f713d7d72cf as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/5160119bc374420ca0dc3f713d7d72cf 2024-11-13T19:26:01,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/5160119bc374420ca0dc3f713d7d72cf, entries=150, sequenceid=212, filesize=11.9 K 2024-11-13T19:26:01,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/c003d6d265024855bc44c9bbda2e7a98 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c003d6d265024855bc44c9bbda2e7a98 2024-11-13T19:26:01,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c003d6d265024855bc44c9bbda2e7a98, entries=150, sequenceid=212, filesize=11.9 K 2024-11-13T19:26:01,931 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for df37f3e4637a53b1d3fb1a07332c3cdb in 858ms, sequenceid=212, compaction requested=true 2024-11-13T19:26:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:A, priority=-2147483648, current under compaction store size is 1 2024-11-13T19:26:01,931 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:26:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:26:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:B, priority=-2147483648, current under compaction store size is 2 2024-11-13T19:26:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:26:01,931 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:26:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store df37f3e4637a53b1d3fb1a07332c3cdb:C, priority=-2147483648, current under compaction store size is 3 2024-11-13T19:26:01,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:26:01,932 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93657 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:26:01,932 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:26:01,932 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/A is initiating minor compaction (all files) 2024-11-13T19:26:01,932 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/B is initiating minor compaction (all files) 2024-11-13T19:26:01,932 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/B in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:01,932 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/A in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:01,932 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e89e471618a943658712d2f175babb31, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/c155d3a5027d4522bb0a238f1da3c668, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/5160119bc374420ca0dc3f713d7d72cf] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=35.9 K 2024-11-13T19:26:01,932 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/3cf2427dbbab48d59b48bcbe82d307bf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bd5bcc90fcba4e48831378ec6d10a516, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/852d22cb75b84233824151b4979b33e3] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=91.5 K 2024-11-13T19:26:01,932 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:01,932 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. files: [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/3cf2427dbbab48d59b48bcbe82d307bf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bd5bcc90fcba4e48831378ec6d10a516, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/852d22cb75b84233824151b4979b33e3] 2024-11-13T19:26:01,932 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting e89e471618a943658712d2f175babb31, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525958213 2024-11-13T19:26:01,932 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cf2427dbbab48d59b48bcbe82d307bf, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525958213 2024-11-13T19:26:01,933 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c155d3a5027d4522bb0a238f1da3c668, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731525958230 2024-11-13T19:26:01,933 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd5bcc90fcba4e48831378ec6d10a516, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731525958230 2024-11-13T19:26:01,933 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 5160119bc374420ca0dc3f713d7d72cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731525960436 2024-11-13T19:26:01,933 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] compactions.Compactor(225): Compacting 852d22cb75b84233824151b4979b33e3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731525960436 2024-11-13T19:26:01,937 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:01,938 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:26:01,938 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/d6e9fefb0d044514942fc38c3de9b942 is 50, key is test_row_0/B:col10/1731525961073/Put/seqid=0 2024-11-13T19:26:01,940 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111371fc856e5232449f8d141c98ced15af9_df37f3e4637a53b1d3fb1a07332c3cdb store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:01,941 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111371fc856e5232449f8d141c98ced15af9_df37f3e4637a53b1d3fb1a07332c3cdb, store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:01,942 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111371fc856e5232449f8d141c98ced15af9_df37f3e4637a53b1d3fb1a07332c3cdb because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:01,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742365_1541 (size=12595) 2024-11-13T19:26:01,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742366_1542 (size=4469) 2024-11-13T19:26:01,946 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#A#compaction#457 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:26:01,947 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/6547eef4c02e422ba8e2c28cf946b71d is 175, key is test_row_0/A:col10/1731525961073/Put/seqid=0 2024-11-13T19:26:01,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742367_1543 (size=31549) 2024-11-13T19:26:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:02,234 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-13T19:26:02,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-13T19:26:02,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113241ae9a2b48b43f29a79437f3b281edc_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525961086/Put/seqid=0 2024-11-13T19:26:02,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742368_1544 (size=12304) 2024-11-13T19:26:02,244 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:26:02,247 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241113241ae9a2b48b43f29a79437f3b281edc_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113241ae9a2b48b43f29a79437f3b281edc_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:02,247 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/dc51649aafca466c84213235808de682, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:02,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/dc51649aafca466c84213235808de682 is 175, key is test_row_0/A:col10/1731525961086/Put/seqid=0 2024-11-13T19:26:02,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742369_1545 (size=31105) 2024-11-13T19:26:02,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526022250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526022250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526022254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526022255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,260 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526022256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,347 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/d6e9fefb0d044514942fc38c3de9b942 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d6e9fefb0d044514942fc38c3de9b942 2024-11-13T19:26:02,350 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/B of df37f3e4637a53b1d3fb1a07332c3cdb into d6e9fefb0d044514942fc38c3de9b942(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:26:02,350 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:02,350 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/B, priority=13, startTime=1731525961931; duration=0sec 2024-11-13T19:26:02,351 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-13T19:26:02,351 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:B 2024-11-13T19:26:02,351 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-13T19:26:02,353 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/6547eef4c02e422ba8e2c28cf946b71d as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/6547eef4c02e422ba8e2c28cf946b71d 2024-11-13T19:26:02,354 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-13T19:26:02,354 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1541): df37f3e4637a53b1d3fb1a07332c3cdb/C is initiating minor compaction (all files) 2024-11-13T19:26:02,354 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of df37f3e4637a53b1d3fb1a07332c3cdb/C in TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:02,354 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/20d4068469594bf595d4ca63c36982b0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/84ed3bf4ada64f9c99815a97a8328f57, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c003d6d265024855bc44c9bbda2e7a98] into tmpdir=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp, totalSize=35.9 K 2024-11-13T19:26:02,354 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 20d4068469594bf595d4ca63c36982b0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731525958213 2024-11-13T19:26:02,355 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting 84ed3bf4ada64f9c99815a97a8328f57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731525958230 2024-11-13T19:26:02,355 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] compactions.Compactor(225): Compacting c003d6d265024855bc44c9bbda2e7a98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731525960436 2024-11-13T19:26:02,356 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/A of df37f3e4637a53b1d3fb1a07332c3cdb into 6547eef4c02e422ba8e2c28cf946b71d(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:26:02,356 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:02,356 INFO [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/A, priority=13, startTime=1731525961931; duration=0sec 2024-11-13T19:26:02,357 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:26:02,357 DEBUG [RS:0;e4b56bc8c4b3:40371-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:A 2024-11-13T19:26:02,360 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): df37f3e4637a53b1d3fb1a07332c3cdb#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T19:26:02,360 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/010836f6b7894242aa933d98d34cd8b0 is 50, key is test_row_0/C:col10/1731525961073/Put/seqid=0 2024-11-13T19:26:02,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742370_1546 (size=12595) 2024-11-13T19:26:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526022362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526022362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526022362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526022362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,368 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/010836f6b7894242aa933d98d34cd8b0 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/010836f6b7894242aa933d98d34cd8b0 2024-11-13T19:26:02,373 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in df37f3e4637a53b1d3fb1a07332c3cdb/C of df37f3e4637a53b1d3fb1a07332c3cdb into 010836f6b7894242aa933d98d34cd8b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T19:26:02,373 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:02,373 INFO [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., storeName=df37f3e4637a53b1d3fb1a07332c3cdb/C, priority=13, startTime=1731525961931; duration=0sec 2024-11-13T19:26:02,373 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-13T19:26:02,373 DEBUG [RS:0;e4b56bc8c4b3:40371-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: df37f3e4637a53b1d3fb1a07332c3cdb:C 2024-11-13T19:26:02,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526022371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,374 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-13T19:26:02,412 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:26:02,412 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-13T19:26:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-13T19:26:02,414 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-13T19:26:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:26:02,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-13T19:26:02,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-13T19:26:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:26:02,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:02,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:02,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526022571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526022571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526022571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526022571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526022582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,653 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/dc51649aafca466c84213235808de682 2024-11-13T19:26:02,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/0f50f967c1094c7abe851d285e5efab1 is 50, key is test_row_0/B:col10/1731525961086/Put/seqid=0 2024-11-13T19:26:02,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742371_1547 (size=12151) 2024-11-13T19:26:02,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:02,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:26:02,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:02,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:02,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526022881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526022881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,883 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526022881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526022881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,884 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:02,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526022891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:03,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:03,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:03,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:26:03,062 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/0f50f967c1094c7abe851d285e5efab1 2024-11-13T19:26:03,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/f648a4bb1d5d4464bf4d7e14353402d3 is 50, key is test_row_0/C:col10/1731525961086/Put/seqid=0 2024-11-13T19:26:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742372_1548 (size=12151) 2024-11-13T19:26:03,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:03,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45838 deadline: 1731526023391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:03,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45818 deadline: 1731526023391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45840 deadline: 1731526023392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:03,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45864 deadline: 1731526023392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:03,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T19:26:03,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40371 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:45816 deadline: 1731526023401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 , the old value is region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-13T19:26:03,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., hostname=e4b56bc8c4b3,40371,1731525792389, seqNum=5 because the exception is null or not the one we care about 2024-11-13T19:26:03,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/f648a4bb1d5d4464bf4d7e14353402d3 2024-11-13T19:26:03,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/dc51649aafca466c84213235808de682 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/dc51649aafca466c84213235808de682 2024-11-13T19:26:03,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/dc51649aafca466c84213235808de682, entries=150, sequenceid=230, filesize=30.4 K 2024-11-13T19:26:03,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/0f50f967c1094c7abe851d285e5efab1 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/0f50f967c1094c7abe851d285e5efab1 2024-11-13T19:26:03,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/0f50f967c1094c7abe851d285e5efab1, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T19:26:03,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/f648a4bb1d5d4464bf4d7e14353402d3 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/f648a4bb1d5d4464bf4d7e14353402d3 2024-11-13T19:26:03,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:03,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:03,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,483 ERROR [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T19:26:03,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/f648a4bb1d5d4464bf4d7e14353402d3, entries=150, sequenceid=230, filesize=11.9 K 2024-11-13T19:26:03,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for df37f3e4637a53b1d3fb1a07332c3cdb in 1253ms, sequenceid=230, compaction requested=false 2024-11-13T19:26:03,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:26:03,636 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40371 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-13T19:26:03,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:03,636 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-13T19:26:03,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138cc4da634e4b44b38207d81816631f69_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525962248/Put/seqid=0 2024-11-13T19:26:03,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742373_1549 (size=12304) 2024-11-13T19:26:04,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:26:04,048 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411138cc4da634e4b44b38207d81816631f69_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138cc4da634e4b44b38207d81816631f69_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/0a9a0c10aee04c399c1c73c8f91de2fe, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/0a9a0c10aee04c399c1c73c8f91de2fe is 175, key is test_row_0/A:col10/1731525962248/Put/seqid=0 2024-11-13T19:26:04,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742374_1550 (size=31105) 2024-11-13T19:26:04,334 INFO [Thread-2216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2216. 2024-11-13T19:26:04,334 DEBUG [Thread-2216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,334 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,335 DEBUG [Thread-2216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,335 INFO [Thread-2208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2208. 2024-11-13T19:26:04,335 DEBUG [Thread-2208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,335 INFO [Thread-2214 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2214. 2024-11-13T19:26:04,335 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,335 DEBUG [Thread-2214 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,335 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,335 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,335 DEBUG [Thread-2214 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,336 INFO [Thread-2210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2210. 2024-11-13T19:26:04,336 DEBUG [Thread-2210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,336 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,336 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,337 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-11-13T19:26:04,337 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,337 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,337 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,403 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. as already flushing 2024-11-13T19:26:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40371 {}] regionserver.HRegion(8855): Flush requested on df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:04,404 INFO [Thread-2197 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2197. 2024-11-13T19:26:04,404 DEBUG [Thread-2197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,404 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,404 DEBUG [Thread-2197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,404 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,414 INFO [Thread-2205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2205. 2024-11-13T19:26:04,414 DEBUG [Thread-2205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,414 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,414 DEBUG [Thread-2205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,414 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,414 INFO [Thread-2201 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2201. 2024-11-13T19:26:04,414 INFO [Thread-2199 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2199. 2024-11-13T19:26:04,414 DEBUG [Thread-2201 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,414 DEBUG [Thread-2199 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,414 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,414 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,415 DEBUG [Thread-2201 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,415 DEBUG [Thread-2199 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,423 INFO [Thread-2203 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2203. 2024-11-13T19:26:04,423 DEBUG [Thread-2203 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-13T19:26:04,423 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,424 DEBUG [Thread-2203 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:04,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:04,461 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/0a9a0c10aee04c399c1c73c8f91de2fe 2024-11-13T19:26:04,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/537709f2adb94be1b4364e64a8153b77 is 50, key is test_row_0/B:col10/1731525962248/Put/seqid=0 2024-11-13T19:26:04,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742375_1551 (size=12151) 2024-11-13T19:26:04,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:26:04,878 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/537709f2adb94be1b4364e64a8153b77 2024-11-13T19:26:04,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/6c1394b4d1234d268da17a199861ffa4 is 50, key is test_row_0/C:col10/1731525962248/Put/seqid=0 2024-11-13T19:26:04,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742376_1552 (size=12151) 2024-11-13T19:26:05,289 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/6c1394b4d1234d268da17a199861ffa4 2024-11-13T19:26:05,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/0a9a0c10aee04c399c1c73c8f91de2fe as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/0a9a0c10aee04c399c1c73c8f91de2fe 2024-11-13T19:26:05,304 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/0a9a0c10aee04c399c1c73c8f91de2fe, entries=150, sequenceid=251, filesize=30.4 K 2024-11-13T19:26:05,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/537709f2adb94be1b4364e64a8153b77 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/537709f2adb94be1b4364e64a8153b77 2024-11-13T19:26:05,309 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/537709f2adb94be1b4364e64a8153b77, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T19:26:05,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/6c1394b4d1234d268da17a199861ffa4 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/6c1394b4d1234d268da17a199861ffa4 2024-11-13T19:26:05,313 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/6c1394b4d1234d268da17a199861ffa4, entries=150, sequenceid=251, filesize=11.9 K 2024-11-13T19:26:05,313 INFO [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=33.54 KB/34350 for df37f3e4637a53b1d3fb1a07332c3cdb in 1677ms, sequenceid=251, compaction requested=true 2024-11-13T19:26:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for df37f3e4637a53b1d3fb1a07332c3cdb: 2024-11-13T19:26:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:05,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/e4b56bc8c4b3:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-13T19:26:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-13T19:26:05,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-13T19:26:05,317 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9000 sec 2024-11-13T19:26:05,318 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 2.9050 sec 2024-11-13T19:26:06,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-13T19:26:06,574 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3995 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3925 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3941 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3997 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3913 2024-11-13T19:26:06,574 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-13T19:26:06,575 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:26:06,575 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:26:06,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:06,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:06,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:06,578 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-13T19:26:06,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-13T19:26:06,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T19:26:06,584 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525966584"}]},"ts":"1731525966584"} 2024-11-13T19:26:06,621 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-13T19:26:06,621 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-13T19:26:06,623 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-13T19:26:06,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, UNASSIGN}] 2024-11-13T19:26:06,628 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, UNASSIGN 2024-11-13T19:26:06,629 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=CLOSING, regionLocation=e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:06,631 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, UNASSIGN because future has completed 2024-11-13T19:26:06,631 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T19:26:06,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389}] 2024-11-13T19:26:06,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T19:26:06,785 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:06,785 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T19:26:06,785 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing df37f3e4637a53b1d3fb1a07332c3cdb, disabling compactions & flushes 2024-11-13T19:26:06,785 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:06,785 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:06,785 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. after waiting 0 ms 2024-11-13T19:26:06,785 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:06,785 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing df37f3e4637a53b1d3fb1a07332c3cdb 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-13T19:26:06,795 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133809228ed2044d2ba8ab002cd51c2aa6_df37f3e4637a53b1d3fb1a07332c3cdb is 50, key is test_row_0/A:col10/1731525964422/Put/seqid=0 2024-11-13T19:26:06,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742377_1553 (size=12404) 2024-11-13T19:26:06,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T19:26:07,200 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T19:26:07,209 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411133809228ed2044d2ba8ab002cd51c2aa6_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133809228ed2044d2ba8ab002cd51c2aa6_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:07,210 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/c30517495a2045b7bd059c689e733e70, store: [table=TestAcidGuarantees family=A region=df37f3e4637a53b1d3fb1a07332c3cdb] 2024-11-13T19:26:07,211 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/c30517495a2045b7bd059c689e733e70 is 175, key is test_row_0/A:col10/1731525964422/Put/seqid=0 2024-11-13T19:26:07,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T19:26:07,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742378_1554 (size=31205) 2024-11-13T19:26:07,617 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/c30517495a2045b7bd059c689e733e70 2024-11-13T19:26:07,629 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/8dda32d43c5044f4ae3175cc2714b86a is 50, key is test_row_0/B:col10/1731525964422/Put/seqid=0 2024-11-13T19:26:07,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742379_1555 (size=12251) 2024-11-13T19:26:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T19:26:08,035 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/8dda32d43c5044f4ae3175cc2714b86a 2024-11-13T19:26:08,046 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/8d60fda0799f48f0aacdc7072d291f50 is 50, key is test_row_0/C:col10/1731525964422/Put/seqid=0 2024-11-13T19:26:08,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742380_1556 (size=12251) 2024-11-13T19:26:08,451 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/8d60fda0799f48f0aacdc7072d291f50 2024-11-13T19:26:08,457 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/A/c30517495a2045b7bd059c689e733e70 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/c30517495a2045b7bd059c689e733e70 2024-11-13T19:26:08,462 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/c30517495a2045b7bd059c689e733e70, entries=150, sequenceid=259, filesize=30.5 K 2024-11-13T19:26:08,463 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/B/8dda32d43c5044f4ae3175cc2714b86a as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/8dda32d43c5044f4ae3175cc2714b86a 2024-11-13T19:26:08,469 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/8dda32d43c5044f4ae3175cc2714b86a, entries=150, sequenceid=259, filesize=12.0 K 2024-11-13T19:26:08,470 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/.tmp/C/8d60fda0799f48f0aacdc7072d291f50 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/8d60fda0799f48f0aacdc7072d291f50 2024-11-13T19:26:08,474 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/8d60fda0799f48f0aacdc7072d291f50, entries=150, sequenceid=259, filesize=12.0 K 2024-11-13T19:26:08,475 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for df37f3e4637a53b1d3fb1a07332c3cdb in 1690ms, sequenceid=259, compaction requested=true 2024-11-13T19:26:08,475 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/b6d1ade74993467a8fc125557c930c91, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/03f418409b194ac48fe7183e5e36db0c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a4cd4ce9d9174d3bb4db40965299efc0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bc118bdb7f224f46ac5063a75fcf5ca9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/df55457959ce42028bbe6bfff478ad97, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/30136a48faf547c590a8b61b5160131e, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/95bf705783294cd8a71510031820042c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a1407250136647b6beae3d55348c9080, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/5ea399eee8de42adba2437cc2afeff24, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/fba8c70a66634b7db803fbbe194cadab, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/3cf2427dbbab48d59b48bcbe82d307bf, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bd5bcc90fcba4e48831378ec6d10a516, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/852d22cb75b84233824151b4979b33e3] to archive 2024-11-13T19:26:08,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:26:08,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/b6d1ade74993467a8fc125557c930c91 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/b6d1ade74993467a8fc125557c930c91 2024-11-13T19:26:08,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/03f418409b194ac48fe7183e5e36db0c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/03f418409b194ac48fe7183e5e36db0c 2024-11-13T19:26:08,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a4cd4ce9d9174d3bb4db40965299efc0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a4cd4ce9d9174d3bb4db40965299efc0 2024-11-13T19:26:08,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bc118bdb7f224f46ac5063a75fcf5ca9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bc118bdb7f224f46ac5063a75fcf5ca9 2024-11-13T19:26:08,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/df55457959ce42028bbe6bfff478ad97 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/df55457959ce42028bbe6bfff478ad97 2024-11-13T19:26:08,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/30136a48faf547c590a8b61b5160131e to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/30136a48faf547c590a8b61b5160131e 2024-11-13T19:26:08,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/95bf705783294cd8a71510031820042c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/95bf705783294cd8a71510031820042c 2024-11-13T19:26:08,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a1407250136647b6beae3d55348c9080 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/a1407250136647b6beae3d55348c9080 2024-11-13T19:26:08,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/5ea399eee8de42adba2437cc2afeff24 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/5ea399eee8de42adba2437cc2afeff24 2024-11-13T19:26:08,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/fba8c70a66634b7db803fbbe194cadab to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/fba8c70a66634b7db803fbbe194cadab 2024-11-13T19:26:08,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/3cf2427dbbab48d59b48bcbe82d307bf to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/3cf2427dbbab48d59b48bcbe82d307bf 2024-11-13T19:26:08,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bd5bcc90fcba4e48831378ec6d10a516 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/bd5bcc90fcba4e48831378ec6d10a516 2024-11-13T19:26:08,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/852d22cb75b84233824151b4979b33e3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/852d22cb75b84233824151b4979b33e3 2024-11-13T19:26:08,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7653bb3c7636422f81f36bb76dcafeed, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/1cb656695e2e4bcf9c88903a4361a816, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e857a81bf02b41baa39105afcf28d920, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/6c03a3c1f6d640c0bfe7ba77cedc79dc, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de4d863c96804019852163a78373d42b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de1a31602f4945d5bf220e2e5bdc7172, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/3972469fc6324c6a99f2d537d21ed98a, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d5266b5755474f87b398a3c9725f910c, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/15409ca872c74ab9a0fadf9546c6a102, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e89e471618a943658712d2f175babb31, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7b7e1fcef4e64e19b18998b081a3178b, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/c155d3a5027d4522bb0a238f1da3c668, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/5160119bc374420ca0dc3f713d7d72cf] to archive 2024-11-13T19:26:08,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:26:08,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7653bb3c7636422f81f36bb76dcafeed to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7653bb3c7636422f81f36bb76dcafeed 2024-11-13T19:26:08,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/1cb656695e2e4bcf9c88903a4361a816 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/1cb656695e2e4bcf9c88903a4361a816 2024-11-13T19:26:08,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e857a81bf02b41baa39105afcf28d920 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e857a81bf02b41baa39105afcf28d920 2024-11-13T19:26:08,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/6c03a3c1f6d640c0bfe7ba77cedc79dc to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/6c03a3c1f6d640c0bfe7ba77cedc79dc 2024-11-13T19:26:08,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de4d863c96804019852163a78373d42b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de4d863c96804019852163a78373d42b 2024-11-13T19:26:08,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de1a31602f4945d5bf220e2e5bdc7172 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/de1a31602f4945d5bf220e2e5bdc7172 2024-11-13T19:26:08,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/3972469fc6324c6a99f2d537d21ed98a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/3972469fc6324c6a99f2d537d21ed98a 2024-11-13T19:26:08,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d5266b5755474f87b398a3c9725f910c to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d5266b5755474f87b398a3c9725f910c 2024-11-13T19:26:08,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/15409ca872c74ab9a0fadf9546c6a102 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/15409ca872c74ab9a0fadf9546c6a102 2024-11-13T19:26:08,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e89e471618a943658712d2f175babb31 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/e89e471618a943658712d2f175babb31 2024-11-13T19:26:08,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7b7e1fcef4e64e19b18998b081a3178b to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/7b7e1fcef4e64e19b18998b081a3178b 2024-11-13T19:26:08,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/c155d3a5027d4522bb0a238f1da3c668 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/c155d3a5027d4522bb0a238f1da3c668 2024-11-13T19:26:08,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/5160119bc374420ca0dc3f713d7d72cf to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/5160119bc374420ca0dc3f713d7d72cf 2024-11-13T19:26:08,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/fefec55da237452b9ff12bf2359d6414, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1269e157f1684be1b8f9172f3841b5a5, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1cd384ee5b14445abf98165891e94fa9, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/47035347b6f34cee8775a499fc935804, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c3101091a3b846d1a91228958b3dc486, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/79111a1f19fb4784903c2e1a2f2411d8, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/2a3bd4bb2c2641b591354ff1a360c153, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/aa6e9b50036440fca3a643c16acf3831, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c769eb07d96e43608ef8fe16b2953107, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/20d4068469594bf595d4ca63c36982b0, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/feff67efeaf64a558779a4b840e259d2, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/84ed3bf4ada64f9c99815a97a8328f57, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c003d6d265024855bc44c9bbda2e7a98] to archive 2024-11-13T19:26:08,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T19:26:08,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/fefec55da237452b9ff12bf2359d6414 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/fefec55da237452b9ff12bf2359d6414 2024-11-13T19:26:08,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1269e157f1684be1b8f9172f3841b5a5 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1269e157f1684be1b8f9172f3841b5a5 2024-11-13T19:26:08,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1cd384ee5b14445abf98165891e94fa9 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/1cd384ee5b14445abf98165891e94fa9 2024-11-13T19:26:08,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/47035347b6f34cee8775a499fc935804 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/47035347b6f34cee8775a499fc935804 2024-11-13T19:26:08,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c3101091a3b846d1a91228958b3dc486 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c3101091a3b846d1a91228958b3dc486 2024-11-13T19:26:08,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/79111a1f19fb4784903c2e1a2f2411d8 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/79111a1f19fb4784903c2e1a2f2411d8 2024-11-13T19:26:08,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/2a3bd4bb2c2641b591354ff1a360c153 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/2a3bd4bb2c2641b591354ff1a360c153 2024-11-13T19:26:08,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/aa6e9b50036440fca3a643c16acf3831 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/aa6e9b50036440fca3a643c16acf3831 2024-11-13T19:26:08,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c769eb07d96e43608ef8fe16b2953107 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c769eb07d96e43608ef8fe16b2953107 2024-11-13T19:26:08,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/20d4068469594bf595d4ca63c36982b0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/20d4068469594bf595d4ca63c36982b0 2024-11-13T19:26:08,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/feff67efeaf64a558779a4b840e259d2 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/feff67efeaf64a558779a4b840e259d2 2024-11-13T19:26:08,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/84ed3bf4ada64f9c99815a97a8328f57 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/84ed3bf4ada64f9c99815a97a8328f57 2024-11-13T19:26:08,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c003d6d265024855bc44c9bbda2e7a98 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/c003d6d265024855bc44c9bbda2e7a98 2024-11-13T19:26:08,545 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/recovered.edits/262.seqid, newMaxSeqId=262, maxSeqId=4 2024-11-13T19:26:08,546 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. 2024-11-13T19:26:08,546 DEBUG [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for df37f3e4637a53b1d3fb1a07332c3cdb: Waiting for close lock at 1731525966785Running coprocessor pre-close hooks at 1731525966785Disabling compacts and flushes for region at 1731525966785Disabling writes for close at 1731525966785Obtaining lock to block concurrent updates at 1731525966785Preparing flush snapshotting stores in df37f3e4637a53b1d3fb1a07332c3cdb at 1731525966786 (+1 ms)Finished memstore snapshotting TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731525966786Flushing stores of TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb. at 1731525966787 (+1 ms)Flushing df37f3e4637a53b1d3fb1a07332c3cdb/A: creating writer at 1731525966787Flushing mob file df37f3e4637a53b1d3fb1a07332c3cdb/A: appending metadata at 1731525966795 (+8 ms)Flushing mob file df37f3e4637a53b1d3fb1a07332c3cdb/A: closing flushed file at 1731525966795Flushing df37f3e4637a53b1d3fb1a07332c3cdb/A: appending metadata at 1731525967210 (+415 ms)Flushing df37f3e4637a53b1d3fb1a07332c3cdb/A: closing flushed file at 1731525967210Flushing df37f3e4637a53b1d3fb1a07332c3cdb/B: creating writer at 1731525967626 (+416 ms)Flushing df37f3e4637a53b1d3fb1a07332c3cdb/B: appending metadata at 1731525967629 (+3 ms)Flushing df37f3e4637a53b1d3fb1a07332c3cdb/B: closing flushed file at 1731525967629Flushing df37f3e4637a53b1d3fb1a07332c3cdb/C: creating writer at 1731525968042 (+413 ms)Flushing df37f3e4637a53b1d3fb1a07332c3cdb/C: appending metadata at 1731525968045 (+3 ms)Flushing df37f3e4637a53b1d3fb1a07332c3cdb/C: closing flushed file at 1731525968045Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b3b2246: reopening flushed file at 1731525968456 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@615809be: reopening flushed file at 1731525968462 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78c5e8bd: reopening flushed file at 1731525968469 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for df37f3e4637a53b1d3fb1a07332c3cdb in 1690ms, sequenceid=259, compaction requested=true at 1731525968475 (+6 ms)Writing region close event to WAL at 1731525968541 (+66 ms)Running coprocessor post-close hooks at 1731525968546 (+5 ms)Closed at 1731525968546 2024-11-13T19:26:08,549 INFO [RS_CLOSE_REGION-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,550 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=df37f3e4637a53b1d3fb1a07332c3cdb, regionState=CLOSED 2024-11-13T19:26:08,552 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 because future has completed 2024-11-13T19:26:08,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-13T19:26:08,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure df37f3e4637a53b1d3fb1a07332c3cdb, server=e4b56bc8c4b3,40371,1731525792389 in 1.9220 sec 2024-11-13T19:26:08,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-13T19:26:08,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=df37f3e4637a53b1d3fb1a07332c3cdb, UNASSIGN in 1.9290 sec 2024-11-13T19:26:08,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-13T19:26:08,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9350 sec 2024-11-13T19:26:08,561 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731525968560"}]},"ts":"1731525968560"} 2024-11-13T19:26:08,562 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-13T19:26:08,562 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-13T19:26:08,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9840 sec 2024-11-13T19:26:08,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-13T19:26:08,734 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:26:08,735 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-13T19:26:08,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:26:08,740 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:26:08,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T19:26:08,742 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:26:08,746 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,750 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C, FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/recovered.edits] 2024-11-13T19:26:08,754 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/0a9a0c10aee04c399c1c73c8f91de2fe to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/0a9a0c10aee04c399c1c73c8f91de2fe 2024-11-13T19:26:08,756 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/6547eef4c02e422ba8e2c28cf946b71d to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/6547eef4c02e422ba8e2c28cf946b71d 2024-11-13T19:26:08,758 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/c30517495a2045b7bd059c689e733e70 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/c30517495a2045b7bd059c689e733e70 2024-11-13T19:26:08,759 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/dc51649aafca466c84213235808de682 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/A/dc51649aafca466c84213235808de682 2024-11-13T19:26:08,762 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/0f50f967c1094c7abe851d285e5efab1 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/0f50f967c1094c7abe851d285e5efab1 2024-11-13T19:26:08,764 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/537709f2adb94be1b4364e64a8153b77 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/537709f2adb94be1b4364e64a8153b77 2024-11-13T19:26:08,766 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/8dda32d43c5044f4ae3175cc2714b86a to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/8dda32d43c5044f4ae3175cc2714b86a 2024-11-13T19:26:08,767 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d6e9fefb0d044514942fc38c3de9b942 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/B/d6e9fefb0d044514942fc38c3de9b942 2024-11-13T19:26:08,771 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/010836f6b7894242aa933d98d34cd8b0 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/010836f6b7894242aa933d98d34cd8b0 2024-11-13T19:26:08,773 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/6c1394b4d1234d268da17a199861ffa4 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/6c1394b4d1234d268da17a199861ffa4 2024-11-13T19:26:08,774 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/8d60fda0799f48f0aacdc7072d291f50 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/8d60fda0799f48f0aacdc7072d291f50 2024-11-13T19:26:08,776 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/f648a4bb1d5d4464bf4d7e14353402d3 to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/C/f648a4bb1d5d4464bf4d7e14353402d3 2024-11-13T19:26:08,780 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/recovered.edits/262.seqid to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb/recovered.edits/262.seqid 2024-11-13T19:26:08,781 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/default/TestAcidGuarantees/df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,781 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-13T19:26:08,782 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:26:08,783 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-13T19:26:08,787 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a26732cb0154ab680d77f2de146b049_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411131a26732cb0154ab680d77f2de146b049_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,789 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113241ae9a2b48b43f29a79437f3b281edc_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113241ae9a2b48b43f29a79437f3b281edc_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,790 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133809228ed2044d2ba8ab002cd51c2aa6_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411133809228ed2044d2ba8ab002cd51c2aa6_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,793 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113424a60f806f84c06824e17ba5c5f94ad_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113424a60f806f84c06824e17ba5c5f94ad_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,795 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347759aae28044be49bdfce01eb114fdc_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111347759aae28044be49bdfce01eb114fdc_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,797 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111363a5c44534a74a38baa1e29c83089df0_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111363a5c44534a74a38baa1e29c83089df0_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,799 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111369803d4675314431a54939ccb6e60fbd_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111369803d4675314431a54939ccb6e60fbd_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,801 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379ef4dd9e17843a3aefdc1a9cb2b0032_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111379ef4dd9e17843a3aefdc1a9cb2b0032_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,802 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113880ff0fa6a6d44fd81427dcedb8ae32d_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113880ff0fa6a6d44fd81427dcedb8ae32d_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,804 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138cc4da634e4b44b38207d81816631f69_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411138cc4da634e4b44b38207d81816631f69_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,805 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113acabf865c55640b2b6b2cf7f625912de_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113acabf865c55640b2b6b2cf7f625912de_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,807 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af2ad894c4c04ad3a725f2445fc9dad4_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113af2ad894c4c04ad3a725f2445fc9dad4_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,808 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c09b196ac6d048179ae2c716596523e4_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113c09b196ac6d048179ae2c716596523e4_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,809 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f74fa18317784821a192bf96993d82d1_df37f3e4637a53b1d3fb1a07332c3cdb to hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241113f74fa18317784821a192bf96993d82d1_df37f3e4637a53b1d3fb1a07332c3cdb 2024-11-13T19:26:08,810 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-13T19:26:08,813 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:26:08,815 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-13T19:26:08,818 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-13T19:26:08,819 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:26:08,820 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-13T19:26:08,820 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731525968820"}]},"ts":"9223372036854775807"} 2024-11-13T19:26:08,822 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-13T19:26:08,822 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => df37f3e4637a53b1d3fb1a07332c3cdb, NAME => 'TestAcidGuarantees,,1731525941156.df37f3e4637a53b1d3fb1a07332c3cdb.', STARTKEY => '', ENDKEY => ''}] 2024-11-13T19:26:08,822 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-13T19:26:08,822 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731525968822"}]},"ts":"9223372036854775807"} 2024-11-13T19:26:08,824 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-13T19:26:08,825 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-13T19:26:08,826 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 90 msec 2024-11-13T19:26:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-13T19:26:08,852 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-13T19:26:08,852 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-13T19:26:08,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 240), OpenFileDescriptor=447 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=540 (was 689), ProcessCount=11 (was 11), AvailableMemoryMB=2548 (was 2560) 2024-11-13T19:26:08,867 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T19:26:08,867 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:26:08,867 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:26:08,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:08,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:08,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T19:26:08,867 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T19:26:08,868 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1975601807, stopped=false 2024-11-13T19:26:08,868 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e4b56bc8c4b3,44929,1731525791607 2024-11-13T19:26:08,915 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T19:26:08,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T19:26:08,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:26:08,916 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:26:08,916 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T19:26:08,916 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T19:26:08,917 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:26:08,917 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:08,918 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e4b56bc8c4b3,40371,1731525792389' ***** 2024-11-13T19:26:08,918 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T19:26:08,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:26:08,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T19:26:08,920 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T19:26:08,920 INFO [RS:0;e4b56bc8c4b3:40371 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T19:26:08,920 INFO [RS:0;e4b56bc8c4b3:40371 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T19:26:08,921 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T19:26:08,921 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(959): stopping server e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:08,921 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:26:08,921 INFO [RS:0;e4b56bc8c4b3:40371 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e4b56bc8c4b3:40371. 2024-11-13T19:26:08,921 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T19:26:08,921 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:08,921 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T19:26:08,921 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T19:26:08,921 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T19:26:08,921 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T19:26:08,922 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T19:26:08,922 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T19:26:08,923 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:08,923 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T19:26:08,923 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T19:26:08,923 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T19:26:08,923 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T19:26:08,923 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T19:26:08,923 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-13T19:26:08,943 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/info/36a9695323a8495792555eef725220d9 is 95, key is TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50./info:/1731525823643/DeleteFamily/seqid=0 2024-11-13T19:26:08,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742381_1557 (size=6514) 2024-11-13T19:26:09,011 INFO [regionserver/e4b56bc8c4b3:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:26:09,123 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:09,280 INFO [regionserver/e4b56bc8c4b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T19:26:09,280 INFO [regionserver/e4b56bc8c4b3:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T19:26:09,323 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:09,347 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/info/36a9695323a8495792555eef725220d9 2024-11-13T19:26:09,376 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/ns/9d527fa294b745708cf2572246e8d3ec is 93, key is TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50./ns:/1731525823600/DeleteFamily/seqid=0 2024-11-13T19:26:09,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742382_1558 (size=5906) 2024-11-13T19:26:09,523 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:09,724 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:09,781 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/ns/9d527fa294b745708cf2572246e8d3ec 2024-11-13T19:26:09,813 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/rep_barrier/d891271aafe44db480bb176acf68ce74 is 102, key is TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50./rep_barrier:/1731525823600/DeleteFamily/seqid=0 2024-11-13T19:26:09,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742383_1559 (size=6025) 2024-11-13T19:26:09,924 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:10,124 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:10,217 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/rep_barrier/d891271aafe44db480bb176acf68ce74 2024-11-13T19:26:10,240 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/table/5fd76f2db91b44a1beb82909676e0316 is 96, key is TestAcidGuarantees,,1731525796909.1334c8616b794fd8c4fb42c4bfbc0d50./table:/1731525823600/DeleteFamily/seqid=0 2024-11-13T19:26:10,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742384_1560 (size=5936) 2024-11-13T19:26:10,324 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:10,525 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T19:26:10,644 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/table/5fd76f2db91b44a1beb82909676e0316 2024-11-13T19:26:10,651 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/info/36a9695323a8495792555eef725220d9 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/info/36a9695323a8495792555eef725220d9 2024-11-13T19:26:10,656 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/info/36a9695323a8495792555eef725220d9, entries=12, sequenceid=90, filesize=6.4 K 2024-11-13T19:26:10,657 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/ns/9d527fa294b745708cf2572246e8d3ec as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/ns/9d527fa294b745708cf2572246e8d3ec 2024-11-13T19:26:10,660 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/ns/9d527fa294b745708cf2572246e8d3ec, entries=8, sequenceid=90, filesize=5.8 K 2024-11-13T19:26:10,661 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/rep_barrier/d891271aafe44db480bb176acf68ce74 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/rep_barrier/d891271aafe44db480bb176acf68ce74 2024-11-13T19:26:10,665 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/rep_barrier/d891271aafe44db480bb176acf68ce74, entries=6, sequenceid=90, filesize=5.9 K 2024-11-13T19:26:10,665 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/.tmp/table/5fd76f2db91b44a1beb82909676e0316 as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/table/5fd76f2db91b44a1beb82909676e0316 2024-11-13T19:26:10,669 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/table/5fd76f2db91b44a1beb82909676e0316, entries=7, sequenceid=90, filesize=5.8 K 2024-11-13T19:26:10,669 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false 2024-11-13T19:26:10,674 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-13T19:26:10,674 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:26:10,675 INFO [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T19:26:10,675 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731525968922Running coprocessor pre-close hooks at 1731525968922Disabling compacts and flushes for region at 1731525968922Disabling writes for close at 1731525968923 (+1 ms)Obtaining lock to block concurrent updates at 1731525968923Preparing flush snapshotting stores in 1588230740 at 1731525968923Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731525968923Flushing stores of hbase:meta,,1.1588230740 at 1731525968924 (+1 ms)Flushing 1588230740/info: creating writer at 1731525968924Flushing 1588230740/info: appending metadata at 1731525968942 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731525968942Flushing 1588230740/ns: creating writer at 1731525969356 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731525969376 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731525969376Flushing 1588230740/rep_barrier: creating writer at 1731525969791 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731525969812 (+21 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731525969812Flushing 1588230740/table: creating writer at 1731525970226 (+414 ms)Flushing 1588230740/table: appending metadata at 1731525970240 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731525970240Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6118ee55: reopening flushed file at 1731525970650 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14290cbb: reopening flushed file at 1731525970656 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d95ee3c: reopening flushed file at 1731525970660 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@468c2200: reopening flushed file at 1731525970665 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false at 1731525970669 (+4 ms)Writing region close event to WAL at 1731525970671 (+2 ms)Running coprocessor post-close hooks at 1731525970674 (+3 ms)Closed at 1731525970675 (+1 ms) 2024-11-13T19:26:10,675 DEBUG [RS_CLOSE_META-regionserver/e4b56bc8c4b3:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T19:26:10,725 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(976): stopping server e4b56bc8c4b3,40371,1731525792389; all regions closed. 2024-11-13T19:26:10,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741834_1010 (size=25344) 2024-11-13T19:26:10,733 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/oldWALs 2024-11-13T19:26:10,733 INFO [RS:0;e4b56bc8c4b3:40371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e4b56bc8c4b3%2C40371%2C1731525792389.meta:.meta(num 1731525796488) 2024-11-13T19:26:10,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741832_1008 (size=12790453) 2024-11-13T19:26:10,738 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/oldWALs 2024-11-13T19:26:10,738 INFO [RS:0;e4b56bc8c4b3:40371 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e4b56bc8c4b3%2C40371%2C1731525792389:(num 1731525795437) 2024-11-13T19:26:10,738 DEBUG [RS:0;e4b56bc8c4b3:40371 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T19:26:10,738 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T19:26:10,738 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:26:10,738 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.ChoreService(370): Chore service for: regionserver/e4b56bc8c4b3:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T19:26:10,739 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:26:10,739 INFO [regionserver/e4b56bc8c4b3:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:26:10,739 INFO [RS:0;e4b56bc8c4b3:40371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40371 2024-11-13T19:26:10,752 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T19:26:10,773 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e4b56bc8c4b3,40371,1731525792389 2024-11-13T19:26:10,773 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T19:26:10,774 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:26:10,782 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e4b56bc8c4b3,40371,1731525792389] 2024-11-13T19:26:10,790 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e4b56bc8c4b3,40371,1731525792389 already deleted, retry=false 2024-11-13T19:26:10,790 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e4b56bc8c4b3,40371,1731525792389 expired; onlineServers=0 2024-11-13T19:26:10,791 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e4b56bc8c4b3,44929,1731525791607' ***** 2024-11-13T19:26:10,791 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T19:26:10,791 INFO [M:0;e4b56bc8c4b3:44929 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T19:26:10,791 INFO [M:0;e4b56bc8c4b3:44929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T19:26:10,791 DEBUG [M:0;e4b56bc8c4b3:44929 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T19:26:10,792 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T19:26:10,792 DEBUG [M:0;e4b56bc8c4b3:44929 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T19:26:10,792 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster-HFileCleaner.small.0-1731525795171 {}] cleaner.HFileCleaner(306): Exit Thread[master/e4b56bc8c4b3:0:becomeActiveMaster-HFileCleaner.small.0-1731525795171,5,FailOnTimeoutGroup] 2024-11-13T19:26:10,792 DEBUG [master/e4b56bc8c4b3:0:becomeActiveMaster-HFileCleaner.large.0-1731525795168 {}] cleaner.HFileCleaner(306): Exit Thread[master/e4b56bc8c4b3:0:becomeActiveMaster-HFileCleaner.large.0-1731525795168,5,FailOnTimeoutGroup] 2024-11-13T19:26:10,792 INFO [M:0;e4b56bc8c4b3:44929 {}] hbase.ChoreService(370): Chore service for: master/e4b56bc8c4b3:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T19:26:10,792 INFO [M:0;e4b56bc8c4b3:44929 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T19:26:10,792 DEBUG [M:0;e4b56bc8c4b3:44929 {}] master.HMaster(1795): Stopping service threads 2024-11-13T19:26:10,792 INFO [M:0;e4b56bc8c4b3:44929 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T19:26:10,792 INFO [M:0;e4b56bc8c4b3:44929 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T19:26:10,792 ERROR [M:0;e4b56bc8c4b3:44929 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:45187 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:45187,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-13T19:26:10,793 INFO [M:0;e4b56bc8c4b3:44929 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T19:26:10,793 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T19:26:10,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T19:26:10,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T19:26:10,798 DEBUG [M:0;e4b56bc8c4b3:44929 {}] zookeeper.ZKUtil(347): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T19:26:10,798 WARN [M:0;e4b56bc8c4b3:44929 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T19:26:10,799 INFO [M:0;e4b56bc8c4b3:44929 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/.lastflushedseqids 2024-11-13T19:26:10,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742385_1561 (size=72) 2024-11-13T19:26:10,883 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:26:10,883 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40371-0x10136f06ec50001, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:26:10,884 INFO [RS:0;e4b56bc8c4b3:40371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:26:10,884 INFO [RS:0;e4b56bc8c4b3:40371 {}] regionserver.HRegionServer(1031): Exiting; stopping=e4b56bc8c4b3,40371,1731525792389; zookeeper connection closed. 2024-11-13T19:26:10,885 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@54064564 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@54064564 2024-11-13T19:26:10,886 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-13T19:26:11,208 INFO [M:0;e4b56bc8c4b3:44929 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T19:26:11,208 INFO [M:0;e4b56bc8c4b3:44929 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T19:26:11,208 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T19:26:11,208 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:26:11,208 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:26:11,208 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T19:26:11,208 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:26:11,208 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=723.25 KB heapSize=886.13 KB 2024-11-13T19:26:11,223 DEBUG [M:0;e4b56bc8c4b3:44929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5eb8f32a4523460984984fa3bb9b130c is 82, key is hbase:meta,,1/info:regioninfo/1731525796579/Put/seqid=0 2024-11-13T19:26:11,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742386_1562 (size=5672) 2024-11-13T19:26:11,627 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2001 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5eb8f32a4523460984984fa3bb9b130c 2024-11-13T19:26:11,685 DEBUG [M:0;e4b56bc8c4b3:44929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0cbd3f8d306d485e8b6467facfc6136e is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x8D/proc:d/1731525944299/Put/seqid=0 2024-11-13T19:26:11,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742387_1563 (size=43500) 2024-11-13T19:26:11,985 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T19:26:11,985 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T19:26:11,986 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-13T19:26:12,089 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=722.69 KB at sequenceid=2001 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0cbd3f8d306d485e8b6467facfc6136e 2024-11-13T19:26:12,094 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0cbd3f8d306d485e8b6467facfc6136e 2024-11-13T19:26:12,113 DEBUG [M:0;e4b56bc8c4b3:44929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d4d215c02e3546fa8aa87bccc586695f is 69, key is e4b56bc8c4b3,40371,1731525792389/rs:state/1731525795190/Put/seqid=0 2024-11-13T19:26:12,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073742388_1564 (size=5156) 2024-11-13T19:26:12,518 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2001 (bloomFilter=true), to=hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d4d215c02e3546fa8aa87bccc586695f 2024-11-13T19:26:12,527 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5eb8f32a4523460984984fa3bb9b130c as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5eb8f32a4523460984984fa3bb9b130c 2024-11-13T19:26:12,533 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5eb8f32a4523460984984fa3bb9b130c, entries=8, sequenceid=2001, filesize=5.5 K 2024-11-13T19:26:12,534 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0cbd3f8d306d485e8b6467facfc6136e as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0cbd3f8d306d485e8b6467facfc6136e 2024-11-13T19:26:12,540 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0cbd3f8d306d485e8b6467facfc6136e 2024-11-13T19:26:12,540 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0cbd3f8d306d485e8b6467facfc6136e, entries=166, sequenceid=2001, filesize=42.5 K 2024-11-13T19:26:12,541 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d4d215c02e3546fa8aa87bccc586695f as hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d4d215c02e3546fa8aa87bccc586695f 2024-11-13T19:26:12,546 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45187/user/jenkins/test-data/7ef9de99-4749-95c4-d2ca-6eca708ff71a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d4d215c02e3546fa8aa87bccc586695f, entries=1, sequenceid=2001, filesize=5.0 K 2024-11-13T19:26:12,547 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(3140): Finished flush of dataSize ~723.25 KB/740605, heapSize ~885.83 KB/907088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1339ms, sequenceid=2001, compaction requested=false 2024-11-13T19:26:12,552 INFO [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T19:26:12,552 DEBUG [M:0;e4b56bc8c4b3:44929 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731525971208Disabling compacts and flushes for region at 1731525971208Disabling writes for close at 1731525971208Obtaining lock to block concurrent updates at 1731525971208Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731525971208Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=740605, getHeapSize=907328, getOffHeapSize=0, getCellsCount=2192 at 1731525971209 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731525971209Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731525971209Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731525971223 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731525971223Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731525971663 (+440 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731525971684 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731525971685 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731525972094 (+409 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731525972112 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731525972112Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@142eb14: reopening flushed file at 1731525972526 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60bb1f7e: reopening flushed file at 1731525972533 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67da003a: reopening flushed file at 1731525972540 (+7 ms)Finished flush of dataSize ~723.25 KB/740605, heapSize ~885.83 KB/907088, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1339ms, sequenceid=2001, compaction requested=false at 1731525972547 (+7 ms)Writing region close event to WAL at 1731525972552 (+5 ms)Closed at 1731525972552 2024-11-13T19:26:12,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38613 is added to blk_1073741830_1006 (size=871853) 2024-11-13T19:26:12,556 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T19:26:12,556 INFO [M:0;e4b56bc8c4b3:44929 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T19:26:12,556 INFO [M:0;e4b56bc8c4b3:44929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44929 2024-11-13T19:26:12,557 INFO [M:0;e4b56bc8c4b3:44929 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T19:26:12,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:26:12,714 INFO [M:0;e4b56bc8c4b3:44929 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T19:26:12,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44929-0x10136f06ec50000, quorum=127.0.0.1:52667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T19:26:12,723 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T19:26:12,726 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:26:12,727 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:26:12,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:26:12,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/hadoop.log.dir/,STOPPED} 2024-11-13T19:26:12,731 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T19:26:12,731 WARN [BP-1151172645-172.17.0.2-1731525788290 heartbeating to localhost/127.0.0.1:45187 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T19:26:12,731 WARN [BP-1151172645-172.17.0.2-1731525788290 heartbeating to localhost/127.0.0.1:45187 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1151172645-172.17.0.2-1731525788290 (Datanode Uuid c432c574-f214-4d93-8597-9db924fa2591) service to localhost/127.0.0.1:45187 2024-11-13T19:26:12,731 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T19:26:12,734 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/data/data1/current/BP-1151172645-172.17.0.2-1731525788290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:26:12,735 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/cluster_541ee53a-1463-d24d-aeb1-44f5c580d6de/data/data2/current/BP-1151172645-172.17.0.2-1731525788290 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T19:26:12,735 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T19:26:12,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T19:26:12,742 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T19:26:12,742 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T19:26:12,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T19:26:12,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b884c398-fd18-7031-1508-0b6bb602f56e/hadoop.log.dir/,STOPPED} 2024-11-13T19:26:12,761 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T19:26:12,872 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down